From: Mike Bayer Date: Mon, 10 Aug 2026 13:08:00 +0000 (-0400) Subject: Establish the fixed set of loader path tokens up front X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=74f0734d1624bf6d587c69557005b5a728574fbf;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Establish the fixed set of loader path tokens up front Fixed issue where unpickling an ORM object that were loaded using loader options making use of wildcard tokens, such as :func:`_orm.load_only` or :func:`_orm.raiseload` with ``"*"``, would fail with ``KeyError`` or ``IndexError`` if the process doing the unpickling had not yet constructed a loader path making use of that same token. This would typically be observed when the object were unpickled in a separate process, such as with the ``spawn`` or ``forkserver`` multiprocessing start methods, the latter of which became the default on POSIX platforms as of Python 3.14. The internal collection of these tokens is now established up front, so that it is identical in every process. Fixed issue where a string ending in ``"*"`` passed to a :class:`_orm.Load` strategy method, such as ``Load(A).joinedload("bs.*")``, would bypass the check which rejects string attribute names in loader options, silently producing a loader path that matched nothing. Such a string now raises :class:`.ArgumentError` with the same message given for any other string attribute name. The bare wildcard ``"*"``, as in ``Load(A).lazyload("*")``, continues to be accepted. Fixes: #13493 Change-Id: I0e8ddf4168bbea132392dd2dfda9a1ef0daf6436 --- diff --git a/doc/build/changelog/unreleased_20/13493.rst b/doc/build/changelog/unreleased_20/13493.rst new file mode 100644 index 0000000000..b1781f3212 --- /dev/null +++ b/doc/build/changelog/unreleased_20/13493.rst @@ -0,0 +1,27 @@ +.. change:: + :tags: bug, orm + :tickets: 13493 + + Fixed issue where unpickling an ORM object that were loaded using loader + options making use of wildcard tokens, such as :func:`_orm.load_only` or + :func:`_orm.raiseload` with ``"*"``, would fail with ``KeyError`` or + ``IndexError`` if the process doing the unpickling had not yet constructed + a loader path making use of that same token. This would typically be + observed when the object were unpickled in a separate process, such as + with the ``spawn`` or ``forkserver`` multiprocessing start methods, the + latter of which became the default on POSIX platforms as of Python 3.14. + The internal collection of these tokens is now established up front, so + that it is identical in every process. + +.. change:: + :tags: bug, orm + :tickets: 13493 + + Fixed issue where a string ending in ``"*"`` passed to a + :class:`_orm.Load` strategy method, such as + ``Load(A).joinedload("bs.*")``, would bypass the check which rejects + string attribute names in loader options, silently producing a loader + path that matched nothing. Such a string now raises + :class:`.ArgumentError` with the same message given for any other + string attribute name. The bare wildcard ``"*"``, as in + ``Load(A).lazyload("*")``, continues to be accepted. diff --git a/lib/sqlalchemy/orm/path_registry.py b/lib/sqlalchemy/orm/path_registry.py index 9a57939db5..6ef126d351 100644 --- a/lib/sqlalchemy/orm/path_registry.py +++ b/lib/sqlalchemy/orm/path_registry.py @@ -15,8 +15,11 @@ import operator from typing import Any from typing import cast from typing import Dict +from typing import Final from typing import Iterator from typing import List +from typing import Literal +from typing import Mapping from typing import Optional from typing import overload from typing import Sequence @@ -86,6 +89,40 @@ def _unreduce_path(path: _SerializedPath) -> PathRegistry: _WILDCARD_TOKEN: _LiteralStar = "*" _DEFAULT_TOKEN = "_sa_default" +_RELATIONSHIP_TOKEN: Final[Literal["relationship"]] = "relationship" +_COLUMN_TOKEN: Final[Literal["column"]] = "column" + +_UNPREFIXED_TOKENS = frozenset([_WILDCARD_TOKEN, _DEFAULT_TOKEN]) +"""the wildcard strings accepted from the user in a loader option. + +these are prefixed with the target property's ``strategy_wildcard_key`` to +form the tokens in :data:`._PATH_TOKENS`, and are not themselves valid as an +element of a path. + +""" + +_PATH_TOKENS = frozenset( + f"{wildcard_key}:{suffix}" + for wildcard_key in (_RELATIONSHIP_TOKEN, _COLUMN_TOKEN) + for suffix in (_WILDCARD_TOKEN, _DEFAULT_TOKEN) +) +"""the complete set of tokens which may appear within a path. + +:attr:`.PathToken._intern` is populated from this collection at module import +time, so that a token is present in every process, including one which has +not yet run any query. + +""" + +_ACCEPTED_TOKENS = _UNPREFIXED_TOKENS | _PATH_TOKENS +"""every string a loader option may accept in place of an attribute name. + +this is the union of the bare wildcards the user writes and the prefixed +tokens which the loader option internals hand back to themselves; only the +latter may appear in a path. + +""" + @inspection._self_inspects class PathRegistry(HasCacheKey): @@ -394,12 +431,12 @@ class _CreatesToken(PathRegistry): is_root: bool def token(self, token: _StrPathToken) -> _TokenRegistry: - if token.endswith(f":{_WILDCARD_TOKEN}"): + if token not in PathToken._intern: + raise exc.ArgumentError(f"invalid token: {token}") + elif token.endswith(f":{_WILDCARD_TOKEN}"): return _TokenRegistry(self, token) - elif token.endswith(f":{_DEFAULT_TOKEN}"): - return _TokenRegistry(self.root, token) else: - raise exc.ArgumentError(f"invalid token: {token}") + return _TokenRegistry(self.root, token) class RootRegistry(_CreatesToken): @@ -446,7 +483,17 @@ PathRegistry.root = RootRegistry() class PathToken(orm_base.InspectionAttr, HasCacheKey, str): """cacheable string token""" - _intern: Dict[str, PathToken] = {} + _intern: Mapping[str, PathToken] + """the :class:`.PathToken` for each of :data:`._PATH_TOKENS`. + + this collection is fully populated below at module import time and is + never added to afterwards; it's typed as :class:`.Mapping` so that a + mutation is flagged by type checkers. + :meth:`.PathRegistry._deserialize_path` relies on it being complete, + distinguishing a token from a mapped attribute key by testing + membership here. + + """ def _gen_cache_key( self, anon_map: anon_map, bindparams: List[BindParameter[Any]] @@ -457,13 +504,8 @@ class PathToken(orm_base.InspectionAttr, HasCacheKey, str): def _path_for_compare(self) -> Optional[_PathRepresentation]: return None - @classmethod - def intern(cls, strvalue: str) -> PathToken: - if strvalue in cls._intern: - return cls._intern[strvalue] - else: - cls._intern[strvalue] = result = PathToken(strvalue) - return result + +PathToken._intern = {token: PathToken(token) for token in _PATH_TOKENS} class _TokenRegistry(PathRegistry): @@ -475,7 +517,7 @@ class _TokenRegistry(PathRegistry): parent: _CreatesToken def __init__(self, parent: _CreatesToken, token: _StrPathToken): - token = PathToken.intern(token) + token = PathToken._intern[token] self.token = token self.parent = parent diff --git a/lib/sqlalchemy/orm/properties.py b/lib/sqlalchemy/orm/properties.py index e260a1b50b..f41ba91d65 100644 --- a/lib/sqlalchemy/orm/properties.py +++ b/lib/sqlalchemy/orm/properties.py @@ -30,7 +30,6 @@ from typing import Union from . import attributes from . import exc as orm_exc -from . import strategy_options from .base import _DeclarativeMapped from .base import class_mapper from .descriptor_props import CompositeProperty @@ -44,6 +43,7 @@ from .interfaces import _MapsColumns from .interfaces import MapperProperty from .interfaces import PropComparator from .interfaces import StrategizedProperty +from .path_registry import _COLUMN_TOKEN from .relationships import RelationshipProperty from .util import de_stringify_annotation from .. import exc as sa_exc @@ -113,7 +113,7 @@ class ColumnProperty( """ - strategy_wildcard_key = strategy_options._COLUMN_TOKEN + strategy_wildcard_key = _COLUMN_TOKEN inherit_cache = True """:meta private:""" diff --git a/lib/sqlalchemy/orm/relationships.py b/lib/sqlalchemy/orm/relationships.py index d02685aa5c..d0f1a135c0 100644 --- a/lib/sqlalchemy/orm/relationships.py +++ b/lib/sqlalchemy/orm/relationships.py @@ -47,7 +47,6 @@ from typing import Union import weakref from . import attributes -from . import strategy_options from ._typing import insp_is_aliased_class from ._typing import is_has_collection_adapter from .base import _DeclarativeMapped @@ -67,6 +66,7 @@ from .interfaces import ONETOMANY from .interfaces import PropComparator from .interfaces import RelationshipDirection from .interfaces import StrategizedProperty +from .path_registry import _RELATIONSHIP_TOKEN from .util import CascadeOptions from .. import exc as sa_exc from .. import Exists @@ -358,7 +358,7 @@ class RelationshipProperty( """ - strategy_wildcard_key = strategy_options._RELATIONSHIP_TOKEN + strategy_wildcard_key = _RELATIONSHIP_TOKEN inherit_cache = True """:meta private:""" diff --git a/lib/sqlalchemy/orm/strategy_options.py b/lib/sqlalchemy/orm/strategy_options.py index 8afdc6e027..60354e6b9b 100644 --- a/lib/sqlalchemy/orm/strategy_options.py +++ b/lib/sqlalchemy/orm/strategy_options.py @@ -15,7 +15,6 @@ from typing import Any from typing import Callable from typing import cast from typing import Dict -from typing import Final from typing import Iterable from typing import Literal from typing import Optional @@ -35,7 +34,10 @@ from .base import entity_str from .base import InspectionAttr from .interfaces import LoaderOption from .path_registry import _AbstractEntityRegistry +from .path_registry import _ACCEPTED_TOKENS +from .path_registry import _COLUMN_TOKEN from .path_registry import _DEFAULT_TOKEN +from .path_registry import _RELATIONSHIP_TOKEN from .path_registry import _StrPathToken from .path_registry import _TokenRegistry from .path_registry import _WILDCARD_TOKEN @@ -55,9 +57,6 @@ from ..sql import visitors from ..sql.base import _generative from ..util.typing import Self -_RELATIONSHIP_TOKEN: Final[Literal["relationship"]] = "relationship" -_COLUMN_TOKEN: Final[Literal["column"]] = "column" - _FN = TypeVar("_FN", bound="Callable[..., Any]") if typing.TYPE_CHECKING: @@ -2202,8 +2201,14 @@ class _TokenStrategyLoad(_LoadElement): ): # assert isinstance(attr, str) or attr is None if attr is not None: - default_token = attr.endswith(_DEFAULT_TOKEN) - if attr.endswith(_WILDCARD_TOKEN) or default_token: + # the only strings accepted here are the wildcard and default + # tokens, either bare or already prefixed with a wildcard key. + # anything else is a leftover from the string based loader + # option API removed in 2.0 and gets the same error as any + # other string. note that testing only for a trailing "*", + # as was formerly the case, lets a name like "addresses.*" + # through to build a loader path that matches nothing + if attr in _ACCEPTED_TOKENS: if wildcard_key: attr = f"{wildcard_key}:{attr}" diff --git a/lib/sqlalchemy/testing/__init__.py b/lib/sqlalchemy/testing/__init__.py index 41ddcedaff..dbda6fdde1 100644 --- a/lib/sqlalchemy/testing/__init__.py +++ b/lib/sqlalchemy/testing/__init__.py @@ -85,6 +85,7 @@ from .util import rowset from .util import run_as_contextmanager from .util import skip_if_timeout from .util import teardown_events +from .util import unpickle_in_subprocess from .warnings import assert_warnings from .warnings import warn_test_suite diff --git a/lib/sqlalchemy/testing/util.py b/lib/sqlalchemy/testing/util.py index 53e130b8c5..05c0d528da 100644 --- a/lib/sqlalchemy/testing/util.py +++ b/lib/sqlalchemy/testing/util.py @@ -15,10 +15,13 @@ import contextlib import decimal import gc from itertools import chain +import os import pickle import random +import subprocess import sys from sys import getsizeof +from tempfile import mkstemp import time import types from typing import Any @@ -62,6 +65,44 @@ def picklers(): yield nt(pickle.loads, lambda d: pickle.dumps(d, protocol)) +def unpickle_in_subprocess(obj, code): + """pickle ``obj`` to a file, then unpickle it in a new interpreter. + + ``code`` is Python source run by that interpreter, which receives the + name of the pickle file as ``sys.argv[1]``. The new interpreter has + the current ``sys.path``, so that the SQLAlchemy under test, as well + as the ``test`` package, are importable. + + Returns the stripped stdout of the subprocess; a non-zero exit status + fails the test, reporting its stderr. + + """ + + fd, filename = mkstemp("pkl") + try: + with os.fdopen(fd, "wb") as file_: + pickle.dump(obj, file_) + + parts = list(sys.path) + if os.environ.get("PYTHONPATH"): + parts.append(os.environ["PYTHONPATH"]) + + proc = subprocess.run( + [sys.executable, "-c", code, filename.replace(os.sep, "/")], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={**os.environ, "PYTHONPATH": os.pathsep.join(parts)}, + ) + finally: + os.unlink(filename) + + if proc.returncode != 0: + raise AssertionError( + "subprocess failed: %s" % proc.stderr.decode(errors="replace") + ) + return proc.stdout.strip() + + def random_choices(population, k=1): return random.choices(population, k=k) diff --git a/test/orm/test_options.py b/test/orm/test_options.py index fb7800a022..89fd81b6d8 100644 --- a/test/orm/test_options.py +++ b/test/orm/test_options.py @@ -437,6 +437,9 @@ class OptionsTest(PathTest, QueryTest): lambda: defer("name"), lambda Address: joinedload("addresses").joinedload(Address.dingaling), lambda: joinedload("addresses"), + # #13493 - a trailing "*" doesn't make a string acceptable + lambda: joinedload("addresses.*"), + lambda: defer("name.*"), ) def test_error_for_string_names_unbound(self, test_case): User, Address = self.classes("User", "Address") @@ -455,6 +458,13 @@ class OptionsTest(PathTest, QueryTest): .joinedload("addresses") .joinedload(Address.dingaling), lambda User: Load(User).joinedload("addresses"), + # #13493 - a trailing "*" doesn't make a string acceptable. these + # were formerly accepted and built a path that matched nothing + lambda User: Load(User).joinedload("addresses.*"), + lambda User: Load(User).defer("name.*"), + lambda User: Load(User).defer("foo:*"), + lambda User: Load(User).lazyload("foo:*"), + lambda User: Load(User).defer("foo:_sa_default"), ) def test_error_for_string_names_bound(self, test_case): User, Address = self.classes("User", "Address") diff --git a/test/orm/test_pickled.py b/test/orm/test_pickled.py index 0c69b2cc86..0926452ac6 100644 --- a/test/orm/test_pickled.py +++ b/test/orm/test_pickled.py @@ -1,29 +1,44 @@ import copy import pickle +import sys import sqlalchemy as sa from sqlalchemy import ForeignKey +from sqlalchemy import inspect from sqlalchemy import Integer from sqlalchemy import MetaData +from sqlalchemy import select from sqlalchemy import String from sqlalchemy import testing from sqlalchemy.orm import aliased from sqlalchemy.orm import attributes from sqlalchemy.orm import clear_mappers from sqlalchemy.orm import collections +from sqlalchemy.orm import defer from sqlalchemy.orm import exc as orm_exc from sqlalchemy.orm import lazyload +from sqlalchemy.orm import Load +from sqlalchemy.orm import load_only +from sqlalchemy.orm import raiseload +from sqlalchemy.orm import registry from sqlalchemy.orm import relationship +from sqlalchemy.orm import selectinload +from sqlalchemy.orm import Session from sqlalchemy.orm import state as sa_state from sqlalchemy.orm import subqueryload +from sqlalchemy.orm import undefer from sqlalchemy.orm import with_loader_criteria from sqlalchemy.orm import with_polymorphic from sqlalchemy.orm.collections import attribute_keyed_dict from sqlalchemy.orm.collections import column_keyed_dict +from sqlalchemy.orm.path_registry import PathRegistry +from sqlalchemy.orm.path_registry import PathToken from sqlalchemy.testing import assert_raises_message from sqlalchemy.testing import eq_ +from sqlalchemy.testing import expect_raises_message from sqlalchemy.testing import fixtures from sqlalchemy.testing import is_not_none +from sqlalchemy.testing import unpickle_in_subprocess from sqlalchemy.testing.fixtures import fixture_session from sqlalchemy.testing.pickleable import Address from sqlalchemy.testing.pickleable import AddressWMixin @@ -666,6 +681,235 @@ class PickleTest(fixtures.MappedTest): pickle.loads(state) +def _token_pickle_tables(metadata): + """tables for :class:`.TokenPathPickleTest`.""" + + users = Table( + "users", + metadata, + Column("id", Integer, primary_key=True, test_needs_autoincrement=True), + Column("name", String(30), nullable=False), + ) + addresses = Table( + "addresses", + metadata, + Column("id", Integer, primary_key=True, test_needs_autoincrement=True), + Column("user_id", None, ForeignKey("users.id")), + Column("email_address", String(50), nullable=False), + ) + return users, addresses + + +def _token_pickle_mappers(reg, users, addresses): + """mapping for :class:`.TokenPathPickleTest`.""" + + reg.map_imperatively( + User, users, properties={"addresses": relationship(Address)} + ) + reg.map_imperatively(Address, addresses) + + +def _unpickle_token_path_main(): + """entry point for the interpreter spawned by + :meth:`.TokenPathPickleTest.test_unpickle_other_process`. + + establishes the same mapping in a process which has not run any query, + then unpickles the instance named in ``sys.argv[1]``. + + """ + + _token_pickle_mappers(registry(), *_token_pickle_tables(MetaData())) + + with open(sys.argv[1], "rb") as file_: + print(pickle.load(file_).name) + + +class TokenPathPickleTest(fixtures.MappedTest): + """test #13493. + + loader paths may include wildcard / default tokens such as + ``"column:*"``, which ride along with a pickled ORM instance and are + reconstructed by :meth:`.PathRegistry.deserialize`. As tokens are + recognized there by their presence in ``PathToken._intern``, that + collection must be fully populated up front, in every process, + including one which has not yet run any query. + + Previously it was populated lazily, so unpickling in a fresh + interpreter failed; this is routinely hit under the ``spawn`` and + ``forkserver`` multiprocessing start methods, the latter of which + became the POSIX default in Python 3.14. + + """ + + run_setup_mappers = "once" + run_inserts = "once" + run_deletes = None + + @classmethod + def define_tables(cls, metadata): + _token_pickle_tables(metadata) + + @classmethod + def setup_mappers(cls): + _token_pickle_mappers( + cls.mapper_registry, cls.tables.users, cls.tables.addresses + ) + + @classmethod + def insert_data(cls, connection): + with Session(connection) as sess: + sess.add( + User( + name="ed", + addresses=[Address(email_address="ed@bar.com")], + ) + ) + sess.commit() + + def test_intern_is_fully_populated(self): + """the complete set of tokens is present at import time. + + this is the actual fix for #13493; the ``load_only()`` / + ``raiseload("*")`` round trips below depend on it. + + """ + + eq_( + set(PathToken._intern), + { + "column:*", + "column:_sa_default", + "relationship:*", + "relationship:_sa_default", + }, + ) + + def test_intern_does_not_grow(self): + """no loader option adds to ``PathToken._intern``. + + the collection is process-global and never pruned, so it must not + accept new entries at runtime, in particular not from paths + arriving via deserialization. + + """ + + before = dict(PathToken._intern) + + with fixture_session() as sess: + for opt in ( + load_only(User.name), + load_only(User.name, raiseload=True), + defer(User.name), + undefer(User.name), + raiseload("*"), + lazyload("*"), + Load(User).defer("*"), + Load(User).raiseload("*"), + selectinload(User.addresses).load_only(Address.email_address), + selectinload(User.addresses).raiseload("*"), + ): + u1 = sess.scalars(select(User).options(opt)).one() + pickle.loads(pickle.dumps(u1)) + sess.expunge_all() + + eq_(PathToken._intern, before) + + def test_unknown_token_rejected(self): + """a string that is not one of the fixed tokens can't create one.""" + + with expect_raises_message( + sa.exc.ArgumentError, "invalid token: column:not_a_token:\\*" + ): + inspect(User)._path_registry.token("column:not_a_token:*") + + @testing.combinations("column", "relationship", argnames="wildcard_key") + @testing.combinations("*", "_sa_default", argnames="token_type") + @testing.combinations(True, False, argnames="from_root") + def test_deserialize_token_path(self, wildcard_key, token_type, from_root): + """every token round trips through serialize() / deserialize().""" + + token = f"{wildcard_key}:{token_type}" + + parent = ( + PathRegistry.root if from_root else inspect(User)._path_registry + ) + path = parent.token(token) + + eq_(PathRegistry.deserialize(path.serialize()).path, path.path) + + def test_pickle_load_only(self): + """load_only() propagates a ``"column:*"`` token within + InstanceState.load_options.""" + + with fixture_session() as sess: + u1 = sess.scalars(select(User).options(load_only(User.name))).one() + sess.expunge_all() + + # prior to the #13493 fix, in a fresh interpreter this raised + # KeyError: 'column:*'; see + # test_unpickle_other_process() for that condition + u2 = pickle.loads(pickle.dumps(u1)) + + eq_( + [ + elem.path.path + for opt in u2._sa_instance_state.load_options + for elem in opt.context + ], + [ + (inspect(User), inspect(User).attrs.name), + (inspect(User), PathToken._intern["column:*"]), + ], + ) + + def test_pickle_wildcard_raiseload(self): + """raiseload("*") leaves a _LoadLazyAttribute in + InstanceState.callables whose loader option has a root level + token path.""" + + with fixture_session() as sess: + u1 = sess.scalars(select(User).options(raiseload("*"))).one() + sess.expunge_all() + + # prior to the #13493 fix, in a fresh interpreter this raised + # IndexError: invalid argument for RootRegistry.__getitem__: None; + # see test_unpickle_other_process() for that condition + u2 = pickle.loads(pickle.dumps(u1)) + + eq_( + u2._sa_instance_state.callables["addresses"].loadopt.path.path, + (PathToken._intern["relationship:_sa_default"],), + ) + + @testing.combinations("load_only", "raiseload", argnames="option") + def test_unpickle_other_process(self, option): + """unpickle in an interpreter that has not itself run a query. + + this is the condition reported in #13493; the tests above run in + the process that produced the pickle, where the tokens would have + been set up by the query itself. + + """ + + if option == "load_only": + opt = load_only(User.name) + else: + opt = raiseload("*") + + with fixture_session() as sess: + u1 = sess.scalars(select(User).options(opt)).one() + sess.expunge_all() + + eq_( + unpickle_in_subprocess( + u1, + "from test.orm.test_pickled import " + "_unpickle_token_path_main; _unpickle_token_path_main()", + ), + b"ed", + ) + + class OptionsTest(_Polymorphic): def test_options_of_type(self): with_poly = with_polymorphic(Person, [Engineer, Manager], flat=True) diff --git a/test/orm/test_utils.py b/test/orm/test_utils.py index 7d9e0a3ac4..9362ac9dcd 100644 --- a/test/orm/test_utils.py +++ b/test/orm/test_utils.py @@ -783,12 +783,12 @@ class PathRegistryTest(_fixtures.FixtureTest): umapper, umapper.attrs.addresses, amapper, - PathToken.intern(":*"), + PathToken._intern["relationship:*"], ) ) is_true(path.is_token) eq_(path[1], umapper.attrs.addresses) - eq_(path[3], ":*") + eq_(path[3], "relationship:*") with expect_raises(IndexError): path[amapper] @@ -801,7 +801,7 @@ class PathRegistryTest(_fixtures.FixtureTest): umapper, umapper.attrs.addresses, amapper, - PathToken.intern(":*"), + PathToken._intern["relationship:*"], ) ) is_true(path.is_token) @@ -921,7 +921,7 @@ class PathRegistryTest(_fixtures.FixtureTest): p2 = PathRegistry.coerce((umapper, umapper.attrs.addresses)) p3 = PathRegistry.coerce((u_alias, umapper.attrs.addresses)) p4 = PathRegistry.coerce((u_alias, umapper.attrs.addresses, amapper)) - p5 = PathRegistry.coerce((u_alias,)).token(":*") + p5 = PathRegistry.coerce((u_alias,)).token("relationship:*") non_object = 54.1432 diff --git a/test/sql/test_resultset.py b/test/sql/test_resultset.py index 8af1fcc538..f64db3a82d 100644 --- a/test/sql/test_resultset.py +++ b/test/sql/test_resultset.py @@ -4,11 +4,7 @@ from contextlib import contextmanager import csv from io import StringIO import operator -import os import pickle -import subprocess -import sys -from tempfile import mkstemp from unittest.mock import Mock from unittest.mock import patch @@ -64,6 +60,7 @@ from sqlalchemy.testing import le_ from sqlalchemy.testing import mock from sqlalchemy.testing import ne_ from sqlalchemy.testing import not_in +from sqlalchemy.testing import unpickle_in_subprocess from sqlalchemy.testing.schema import Column from sqlalchemy.testing.schema import Table @@ -571,27 +568,14 @@ class CursorResultTest(fixtures.TablesTest): def test_pickle_rows_other_process(self, connection, use_labels): result = self._pickle_row_data(connection, use_labels) - f, name = mkstemp("pkl") - with os.fdopen(f, "wb") as f: - pickle.dump(result, f) - name = name.replace(os.sep, "/") code = ( - "import sqlalchemy; import pickle; print([" - f"r[0] for r in pickle.load(open('''{name}''', 'rb'))])" - ) - parts = list(sys.path) - if os.environ.get("PYTHONPATH"): - parts.append(os.environ["PYTHONPATH"]) - pythonpath = os.pathsep.join(parts) - proc = subprocess.run( - [sys.executable, "-c", code], - stdout=subprocess.PIPE, - env={**os.environ, "PYTHONPATH": pythonpath}, - ) - exp = str([r[0] for r in result]).encode() - eq_(proc.returncode, 0) - eq_(proc.stdout.strip(), exp) - os.unlink(name) + "import sqlalchemy; import pickle; import sys; print(" + "[r[0] for r in pickle.load(open(sys.argv[1], 'rb'))])" + ) + eq_( + unpickle_in_subprocess(result, code), + str([r[0] for r in result]).encode(), + ) def test_column_error_printing(self, connection): result = connection.execute(select(1)) diff --git a/test/sql/test_types.py b/test/sql/test_types.py index 699600a406..f5dd5a0e17 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -3,10 +3,6 @@ import decimal import importlib import operator import os -import pickle -import subprocess -import sys -from tempfile import mkstemp import uuid import sqlalchemy as sa @@ -97,6 +93,7 @@ from sqlalchemy.testing import is_not from sqlalchemy.testing import is_true from sqlalchemy.testing import mock from sqlalchemy.testing import pickleable +from sqlalchemy.testing import unpickle_in_subprocess from sqlalchemy.testing.assertions import expect_raises_message from sqlalchemy.testing.schema import Column from sqlalchemy.testing.schema import pep435_enum @@ -778,27 +775,12 @@ class PickleTypesTest(fixtures.TestBase): meta = MetaData() Table("foo", meta, column_type) + code = ( + "import sqlalchemy; import pickle; import sys; " + "pickle.load(open(sys.argv[1], 'rb'))" + ) for target in column_type, meta: - f, name = mkstemp("pkl") - with os.fdopen(f, "wb") as f: - pickle.dump(target, f) - - name = name.replace(os.sep, "/") - code = ( - "import sqlalchemy; import pickle; " - f"pickle.load(open('''{name}''', 'rb'))" - ) - parts = list(sys.path) - if os.environ.get("PYTHONPATH"): - parts.append(os.environ["PYTHONPATH"]) - pythonpath = os.pathsep.join(parts) - proc = subprocess.run( - [sys.executable, "-c", code], - env={**os.environ, "PYTHONPATH": pythonpath}, - stderr=subprocess.PIPE, - ) - eq_(proc.returncode, 0, proc.stderr.decode(errors="replace")) - os.unlink(name) + unpickle_in_subprocess(target, code) class _UserDefinedTypeFixture: