--- /dev/null
+.. change::
+ :tags: bug, orm
+ :tickets: 13493
+
+ Fixed issue where unpickling an ORM object that were loaded using loader
+ options making use of wildcard tokens, such as :func:`_orm.load_only` or
+ :func:`_orm.raiseload` with ``"*"``, would fail with ``KeyError`` or
+ ``IndexError`` if the process doing the unpickling had not yet constructed
+ a loader path making use of that same token. This would typically be
+ observed when the object were unpickled in a separate process, such as
+ with the ``spawn`` or ``forkserver`` multiprocessing start methods, the
+ latter of which became the default on POSIX platforms as of Python 3.14.
+ The internal collection of these tokens is now established up front, so
+ that it is identical in every process.
+
+.. change::
+ :tags: bug, orm
+ :tickets: 13493
+
+ Fixed issue where a string ending in ``"*"`` passed to a
+ :class:`_orm.Load` strategy method, such as
+ ``Load(A).joinedload("bs.*")``, would bypass the check which rejects
+ string attribute names in loader options, silently producing a loader
+ path that matched nothing. Such a string now raises
+ :class:`.ArgumentError` with the same message given for any other
+ string attribute name. The bare wildcard ``"*"``, as in
+ ``Load(A).lazyload("*")``, continues to be accepted.
from typing import Dict
from typing import Iterator
from typing import List
+from typing import Mapping
from typing import Optional
from typing import overload
from typing import Sequence
from .. import util
from ..sql import visitors
from ..sql.cache_key import HasCacheKey
+from ..util.typing import Final
+from ..util.typing import Literal
if TYPE_CHECKING:
from ._typing import _InternalEntityType
_WILDCARD_TOKEN: _LiteralStar = "*"
_DEFAULT_TOKEN = "_sa_default"
+_RELATIONSHIP_TOKEN: Final[Literal["relationship"]] = "relationship"
+_COLUMN_TOKEN: Final[Literal["column"]] = "column"
+
+_UNPREFIXED_TOKENS = frozenset([_WILDCARD_TOKEN, _DEFAULT_TOKEN])
+"""the wildcard strings accepted from the user in a loader option.
+
+these are prefixed with the target property's ``strategy_wildcard_key`` to
+form the tokens in :data:`._PATH_TOKENS`, and are not themselves valid as an
+element of a path.
+
+"""
+
+_PATH_TOKENS = frozenset(
+ f"{wildcard_key}:{suffix}"
+ for wildcard_key in (_RELATIONSHIP_TOKEN, _COLUMN_TOKEN)
+ for suffix in (_WILDCARD_TOKEN, _DEFAULT_TOKEN)
+)
+"""the complete set of tokens which may appear within a path.
+
+:attr:`.PathToken._intern` is populated from this collection at module import
+time, so that a token is present in every process, including one which has
+not yet run any query.
+
+"""
+
+_ACCEPTED_TOKENS = _UNPREFIXED_TOKENS | _PATH_TOKENS
+"""every string a loader option may accept in place of an attribute name.
+
+this is the union of the bare wildcards the user writes and the prefixed
+tokens which the loader option internals hand back to themselves; only the
+latter may appear in a path.
+
+"""
+
class PathRegistry(HasCacheKey):
"""Represent query load paths and registry functions.
is_root: bool
def token(self, token: _StrPathToken) -> TokenRegistry:
- if token.endswith(f":{_WILDCARD_TOKEN}"):
+ if token not in PathToken._intern:
+ raise exc.ArgumentError(f"invalid token: {token}")
+ elif token.endswith(f":{_WILDCARD_TOKEN}"):
return TokenRegistry(self, token)
- elif token.endswith(f":{_DEFAULT_TOKEN}"):
- return TokenRegistry(self.root, token)
else:
- raise exc.ArgumentError(f"invalid token: {token}")
+ return TokenRegistry(self.root, token)
class RootRegistry(CreatesToken):
class PathToken(orm_base.InspectionAttr, HasCacheKey, str):
"""cacheable string token"""
- _intern: Dict[str, PathToken] = {}
+ _intern: Mapping[str, PathToken]
+ """the :class:`.PathToken` for each of :data:`._PATH_TOKENS`.
+
+ this collection is fully populated below at module import time and is
+ never added to afterwards; it's typed as :class:`.Mapping` so that a
+ mutation is flagged by type checkers.
+ :meth:`.PathRegistry._deserialize_path` relies on it being complete,
+ distinguishing a token from a mapped attribute key by testing
+ membership here.
+
+ """
def _gen_cache_key(
self, anon_map: anon_map, bindparams: List[BindParameter[Any]]
def _path_for_compare(self) -> Optional[_PathRepresentation]:
return None
- @classmethod
- def intern(cls, strvalue: str) -> PathToken:
- if strvalue in cls._intern:
- return cls._intern[strvalue]
- else:
- cls._intern[strvalue] = result = PathToken(strvalue)
- return result
+
+PathToken._intern = {token: PathToken(token) for token in _PATH_TOKENS}
class TokenRegistry(PathRegistry):
parent: CreatesToken
def __init__(self, parent: CreatesToken, token: _StrPathToken):
- token = PathToken.intern(token)
+ token = PathToken._intern[token]
self.token = token
self.parent = parent
from . import attributes
from . import exc as orm_exc
-from . import strategy_options
from .base import _DeclarativeMapped
from .base import class_mapper
from .descriptor_props import CompositeProperty
from .interfaces import MapperProperty
from .interfaces import PropComparator
from .interfaces import StrategizedProperty
+from .path_registry import _COLUMN_TOKEN
from .relationships import RelationshipProperty
from .util import de_stringify_annotation
from .. import exc as sa_exc
"""
- strategy_wildcard_key = strategy_options._COLUMN_TOKEN
+ strategy_wildcard_key = _COLUMN_TOKEN
inherit_cache = True
""":meta private:"""
import weakref
from . import attributes
-from . import strategy_options
from ._typing import insp_is_aliased_class
from ._typing import is_has_collection_adapter
from .base import _DeclarativeMapped
from .interfaces import PropComparator
from .interfaces import RelationshipDirection
from .interfaces import StrategizedProperty
+from .path_registry import _RELATIONSHIP_TOKEN
from .util import _orm_annotate
from .util import _orm_deannotate
from .util import CascadeOptions
"""
- strategy_wildcard_key = strategy_options._RELATIONSHIP_TOKEN
+ strategy_wildcard_key = _RELATIONSHIP_TOKEN
inherit_cache = True
""":meta private:"""
from .attributes import QueryableAttribute
from .base import InspectionAttr
from .interfaces import LoaderOption
+from .path_registry import _ACCEPTED_TOKENS
+from .path_registry import _COLUMN_TOKEN
from .path_registry import _DEFAULT_TOKEN
+from .path_registry import _RELATIONSHIP_TOKEN
from .path_registry import _StrPathToken
from .path_registry import _WILDCARD_TOKEN
from .path_registry import AbstractEntityRegistry
from ..sql import traversals
from ..sql import visitors
from ..sql.base import _generative
-from ..util.typing import Final
from ..util.typing import Literal
from ..util.typing import Self
-_RELATIONSHIP_TOKEN: Final[Literal["relationship"]] = "relationship"
-_COLUMN_TOKEN: Final[Literal["column"]] = "column"
-
_FN = TypeVar("_FN", bound="Callable[..., Any]")
if typing.TYPE_CHECKING:
):
# assert isinstance(attr, str) or attr is None
if attr is not None:
- default_token = attr.endswith(_DEFAULT_TOKEN)
- if attr.endswith(_WILDCARD_TOKEN) or default_token:
+ # the only strings accepted here are the wildcard and default
+ # tokens, either bare or already prefixed with a wildcard key.
+ # anything else is a leftover from the string based loader
+ # option API removed in 2.0 and gets the same error as any
+ # other string. note that testing only for a trailing "*",
+ # as was formerly the case, lets a name like "addresses.*"
+ # through to build a loader path that matches nothing
+ if attr in _ACCEPTED_TOKENS:
if wildcard_key:
attr = f"{wildcard_key}:{attr}"
from .util import run_as_contextmanager
from .util import skip_if_timeout
from .util import teardown_events
+from .util import unpickle_in_subprocess
from .warnings import assert_warnings
from .warnings import warn_test_suite
import decimal
import gc
from itertools import chain
+import os
import pickle
import random
+import subprocess
import sys
from sys import getsizeof
+from tempfile import mkstemp
import time
import types
from typing import Any
yield nt(pickle.loads, lambda d: pickle.dumps(d, protocol))
+def unpickle_in_subprocess(obj, code):
+ """pickle ``obj`` to a file, then unpickle it in a new interpreter.
+
+ ``code`` is Python source run by that interpreter, which receives the
+ name of the pickle file as ``sys.argv[1]``. The new interpreter has
+ the current ``sys.path``, so that the SQLAlchemy under test, as well
+ as the ``test`` package, are importable.
+
+ Returns the stripped stdout of the subprocess; a non-zero exit status
+ fails the test, reporting its stderr.
+
+ """
+
+ fd, filename = mkstemp("pkl")
+ try:
+ with os.fdopen(fd, "wb") as file_:
+ pickle.dump(obj, file_)
+
+ parts = list(sys.path)
+ if os.environ.get("PYTHONPATH"):
+ parts.append(os.environ["PYTHONPATH"])
+
+ proc = subprocess.run(
+ [sys.executable, "-c", code, filename.replace(os.sep, "/")],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env={**os.environ, "PYTHONPATH": os.pathsep.join(parts)},
+ )
+ finally:
+ os.unlink(filename)
+
+ if proc.returncode != 0:
+ raise AssertionError(
+ "subprocess failed: %s" % proc.stderr.decode(errors="replace")
+ )
+ return proc.stdout.strip()
+
+
def random_choices(population, k=1):
return random.choices(population, k=k)
lambda: defer("name"),
lambda Address: joinedload("addresses").joinedload(Address.dingaling),
lambda: joinedload("addresses"),
+ # #13493 - a trailing "*" doesn't make a string acceptable
+ lambda: joinedload("addresses.*"),
+ lambda: defer("name.*"),
)
def test_error_for_string_names_unbound(self, test_case):
User, Address = self.classes("User", "Address")
.joinedload("addresses")
.joinedload(Address.dingaling),
lambda User: Load(User).joinedload("addresses"),
+ # #13493 - a trailing "*" doesn't make a string acceptable. these
+ # were formerly accepted and built a path that matched nothing
+ lambda User: Load(User).joinedload("addresses.*"),
+ lambda User: Load(User).defer("name.*"),
+ lambda User: Load(User).defer("foo:*"),
+ lambda User: Load(User).lazyload("foo:*"),
+ lambda User: Load(User).defer("foo:_sa_default"),
)
def test_error_for_string_names_bound(self, test_case):
User, Address = self.classes("User", "Address")
import copy
import pickle
+import sys
import sqlalchemy as sa
from sqlalchemy import ForeignKey
+from sqlalchemy import inspect
from sqlalchemy import Integer
from sqlalchemy import MetaData
+from sqlalchemy import select
from sqlalchemy import String
from sqlalchemy import testing
from sqlalchemy.orm import aliased
from sqlalchemy.orm import attributes
from sqlalchemy.orm import clear_mappers
from sqlalchemy.orm import collections
+from sqlalchemy.orm import defer
from sqlalchemy.orm import exc as orm_exc
from sqlalchemy.orm import lazyload
+from sqlalchemy.orm import Load
+from sqlalchemy.orm import load_only
+from sqlalchemy.orm import raiseload
+from sqlalchemy.orm import registry
from sqlalchemy.orm import relationship
+from sqlalchemy.orm import selectinload
+from sqlalchemy.orm import Session
from sqlalchemy.orm import state as sa_state
from sqlalchemy.orm import subqueryload
+from sqlalchemy.orm import undefer
from sqlalchemy.orm import with_loader_criteria
from sqlalchemy.orm import with_polymorphic
from sqlalchemy.orm.collections import attribute_keyed_dict
from sqlalchemy.orm.collections import column_keyed_dict
+from sqlalchemy.orm.path_registry import PathRegistry
+from sqlalchemy.orm.path_registry import PathToken
from sqlalchemy.testing import assert_raises_message
from sqlalchemy.testing import eq_
+from sqlalchemy.testing import expect_raises_message
from sqlalchemy.testing import fixtures
from sqlalchemy.testing import is_not_none
+from sqlalchemy.testing import unpickle_in_subprocess
from sqlalchemy.testing.fixtures import fixture_session
from sqlalchemy.testing.pickleable import Address
from sqlalchemy.testing.pickleable import AddressWMixin
pickle.loads(state)
+def _token_pickle_tables(metadata):
+ """tables for :class:`.TokenPathPickleTest`."""
+
+ users = Table(
+ "users",
+ metadata,
+ Column("id", Integer, primary_key=True, test_needs_autoincrement=True),
+ Column("name", String(30), nullable=False),
+ )
+ addresses = Table(
+ "addresses",
+ metadata,
+ Column("id", Integer, primary_key=True, test_needs_autoincrement=True),
+ Column("user_id", None, ForeignKey("users.id")),
+ Column("email_address", String(50), nullable=False),
+ )
+ return users, addresses
+
+
+def _token_pickle_mappers(reg, users, addresses):
+ """mapping for :class:`.TokenPathPickleTest`."""
+
+ reg.map_imperatively(
+ User, users, properties={"addresses": relationship(Address)}
+ )
+ reg.map_imperatively(Address, addresses)
+
+
+def _unpickle_token_path_main():
+ """entry point for the interpreter spawned by
+ :meth:`.TokenPathPickleTest.test_unpickle_other_process`.
+
+ establishes the same mapping in a process which has not run any query,
+ then unpickles the instance named in ``sys.argv[1]``.
+
+ """
+
+ _token_pickle_mappers(registry(), *_token_pickle_tables(MetaData()))
+
+ with open(sys.argv[1], "rb") as file_:
+ print(pickle.load(file_).name)
+
+
+class TokenPathPickleTest(fixtures.MappedTest):
+ """test #13493.
+
+ loader paths may include wildcard / default tokens such as
+ ``"column:*"``, which ride along with a pickled ORM instance and are
+ reconstructed by :meth:`.PathRegistry.deserialize`. As tokens are
+ recognized there by their presence in ``PathToken._intern``, that
+ collection must be fully populated up front, in every process,
+ including one which has not yet run any query.
+
+ Previously it was populated lazily, so unpickling in a fresh
+ interpreter failed; this is routinely hit under the ``spawn`` and
+ ``forkserver`` multiprocessing start methods, the latter of which
+ became the POSIX default in Python 3.14.
+
+ """
+
+ run_setup_mappers = "once"
+ run_inserts = "once"
+ run_deletes = None
+
+ @classmethod
+ def define_tables(cls, metadata):
+ _token_pickle_tables(metadata)
+
+ @classmethod
+ def setup_mappers(cls):
+ _token_pickle_mappers(
+ cls.mapper_registry, cls.tables.users, cls.tables.addresses
+ )
+
+ @classmethod
+ def insert_data(cls, connection):
+ with Session(connection) as sess:
+ sess.add(
+ User(
+ name="ed",
+ addresses=[Address(email_address="ed@bar.com")],
+ )
+ )
+ sess.commit()
+
+ def test_intern_is_fully_populated(self):
+ """the complete set of tokens is present at import time.
+
+ this is the actual fix for #13493; the ``load_only()`` /
+ ``raiseload("*")`` round trips below depend on it.
+
+ """
+
+ eq_(
+ set(PathToken._intern),
+ {
+ "column:*",
+ "column:_sa_default",
+ "relationship:*",
+ "relationship:_sa_default",
+ },
+ )
+
+ def test_intern_does_not_grow(self):
+ """no loader option adds to ``PathToken._intern``.
+
+ the collection is process-global and never pruned, so it must not
+ accept new entries at runtime, in particular not from paths
+ arriving via deserialization.
+
+ """
+
+ before = dict(PathToken._intern)
+
+ with fixture_session() as sess:
+ for opt in (
+ load_only(User.name),
+ load_only(User.name, raiseload=True),
+ defer(User.name),
+ undefer(User.name),
+ raiseload("*"),
+ lazyload("*"),
+ Load(User).defer("*"),
+ Load(User).raiseload("*"),
+ selectinload(User.addresses).load_only(Address.email_address),
+ selectinload(User.addresses).raiseload("*"),
+ ):
+ u1 = sess.scalars(select(User).options(opt)).one()
+ pickle.loads(pickle.dumps(u1))
+ sess.expunge_all()
+
+ eq_(PathToken._intern, before)
+
+ def test_unknown_token_rejected(self):
+ """a string that is not one of the fixed tokens can't create one."""
+
+ with expect_raises_message(
+ sa.exc.ArgumentError, "invalid token: column:not_a_token:\\*"
+ ):
+ inspect(User)._path_registry.token("column:not_a_token:*")
+
+ @testing.combinations("column", "relationship", argnames="wildcard_key")
+ @testing.combinations("*", "_sa_default", argnames="token_type")
+ @testing.combinations(True, False, argnames="from_root")
+ def test_deserialize_token_path(self, wildcard_key, token_type, from_root):
+ """every token round trips through serialize() / deserialize()."""
+
+ token = f"{wildcard_key}:{token_type}"
+
+ parent = (
+ PathRegistry.root if from_root else inspect(User)._path_registry
+ )
+ path = parent.token(token)
+
+ eq_(PathRegistry.deserialize(path.serialize()).path, path.path)
+
+ def test_pickle_load_only(self):
+ """load_only() propagates a ``"column:*"`` token within
+ InstanceState.load_options."""
+
+ with fixture_session() as sess:
+ u1 = sess.scalars(select(User).options(load_only(User.name))).one()
+ sess.expunge_all()
+
+ # prior to the #13493 fix, in a fresh interpreter this raised
+ # KeyError: 'column:*'; see
+ # test_unpickle_other_process() for that condition
+ u2 = pickle.loads(pickle.dumps(u1))
+
+ eq_(
+ [
+ elem.path.path
+ for opt in u2._sa_instance_state.load_options
+ for elem in opt.context
+ ],
+ [
+ (inspect(User), inspect(User).attrs.name),
+ (inspect(User), PathToken._intern["column:*"]),
+ ],
+ )
+
+ def test_pickle_wildcard_raiseload(self):
+ """raiseload("*") leaves a _LoadLazyAttribute in
+ InstanceState.callables whose loader option has a root level
+ token path."""
+
+ with fixture_session() as sess:
+ u1 = sess.scalars(select(User).options(raiseload("*"))).one()
+ sess.expunge_all()
+
+ # prior to the #13493 fix, in a fresh interpreter this raised
+ # IndexError: invalid argument for RootRegistry.__getitem__: None;
+ # see test_unpickle_other_process() for that condition
+ u2 = pickle.loads(pickle.dumps(u1))
+
+ eq_(
+ u2._sa_instance_state.callables["addresses"].loadopt.path.path,
+ (PathToken._intern["relationship:_sa_default"],),
+ )
+
+ @testing.combinations("load_only", "raiseload", argnames="option")
+ def test_unpickle_other_process(self, option):
+ """unpickle in an interpreter that has not itself run a query.
+
+ this is the condition reported in #13493; the tests above run in
+ the process that produced the pickle, where the tokens would have
+ been set up by the query itself.
+
+ """
+
+ if option == "load_only":
+ opt = load_only(User.name)
+ else:
+ opt = raiseload("*")
+
+ with fixture_session() as sess:
+ u1 = sess.scalars(select(User).options(opt)).one()
+ sess.expunge_all()
+
+ eq_(
+ unpickle_in_subprocess(
+ u1,
+ "from test.orm.test_pickled import "
+ "_unpickle_token_path_main; _unpickle_token_path_main()",
+ ),
+ b"ed",
+ )
+
+
class OptionsTest(_Polymorphic):
def test_options_of_type(self):
with_poly = with_polymorphic(Person, [Engineer, Manager], flat=True)
umapper,
umapper.attrs.addresses,
amapper,
- PathToken.intern(":*"),
+ PathToken._intern["relationship:*"],
)
)
is_true(path.is_token)
eq_(path[1], umapper.attrs.addresses)
- eq_(path[3], ":*")
+ eq_(path[3], "relationship:*")
with expect_raises(IndexError):
path[amapper]
umapper,
umapper.attrs.addresses,
amapper,
- PathToken.intern(":*"),
+ PathToken._intern["relationship:*"],
)
)
is_true(path.is_token)
p2 = PathRegistry.coerce((umapper, umapper.attrs.addresses))
p3 = PathRegistry.coerce((u_alias, umapper.attrs.addresses))
p4 = PathRegistry.coerce((u_alias, umapper.attrs.addresses, amapper))
- p5 = PathRegistry.coerce((u_alias,)).token(":*")
+ p5 = PathRegistry.coerce((u_alias,)).token("relationship:*")
non_object = 54.1432
import csv
from io import StringIO
import operator
-import os
import pickle
-import subprocess
-import sys
-from tempfile import mkstemp
from unittest.mock import Mock
from unittest.mock import patch
from sqlalchemy.testing import mock
from sqlalchemy.testing import ne_
from sqlalchemy.testing import not_in
+from sqlalchemy.testing import unpickle_in_subprocess
from sqlalchemy.testing.schema import Column
from sqlalchemy.testing.schema import Table
def test_pickle_rows_other_process(self, connection, use_labels):
result = self._pickle_row_data(connection, use_labels)
- f, name = mkstemp("pkl")
- with os.fdopen(f, "wb") as f:
- pickle.dump(result, f)
- name = name.replace(os.sep, "/")
code = (
- "import sqlalchemy; import pickle; print(["
- f"r[0] for r in pickle.load(open('''{name}''', 'rb'))])"
- )
- parts = list(sys.path)
- if os.environ.get("PYTHONPATH"):
- parts.append(os.environ["PYTHONPATH"])
- pythonpath = os.pathsep.join(parts)
- proc = subprocess.run(
- [sys.executable, "-c", code],
- stdout=subprocess.PIPE,
- env={**os.environ, "PYTHONPATH": pythonpath},
- )
- exp = str([r[0] for r in result]).encode()
- eq_(proc.returncode, 0)
- eq_(proc.stdout.strip(), exp)
- os.unlink(name)
+ "import sqlalchemy; import pickle; import sys; print("
+ "[r[0] for r in pickle.load(open(sys.argv[1], 'rb'))])"
+ )
+ eq_(
+ unpickle_in_subprocess(result, code),
+ str([r[0] for r in result]).encode(),
+ )
def test_column_error_printing(self, connection):
result = connection.execute(select(1))
import importlib
import operator
import os
-import pickle
-import subprocess
-import sys
-from tempfile import mkstemp
import sqlalchemy as sa
from sqlalchemy import and_
from sqlalchemy.testing import is_true
from sqlalchemy.testing import mock
from sqlalchemy.testing import pickleable
+from sqlalchemy.testing import unpickle_in_subprocess
from sqlalchemy.testing.assertions import expect_raises_message
from sqlalchemy.testing.schema import Column
from sqlalchemy.testing.schema import pep435_enum
meta = MetaData()
Table("foo", meta, column_type)
+ code = (
+ "import sqlalchemy; import pickle; import sys; "
+ "pickle.load(open(sys.argv[1], 'rb'))"
+ )
for target in column_type, meta:
- f, name = mkstemp("pkl")
- with os.fdopen(f, "wb") as f:
- pickle.dump(target, f)
-
- name = name.replace(os.sep, "/")
- code = (
- "import sqlalchemy; import pickle; "
- f"pickle.load(open('''{name}''', 'rb'))"
- )
- parts = list(sys.path)
- if os.environ.get("PYTHONPATH"):
- parts.append(os.environ["PYTHONPATH"])
- pythonpath = os.pathsep.join(parts)
- proc = subprocess.run(
- [sys.executable, "-c", code],
- env={**os.environ, "PYTHONPATH": pythonpath},
- stderr=subprocess.PIPE,
- )
- eq_(proc.returncode, 0, proc.stderr.decode(errors="replace"))
- os.unlink(name)
+ unpickle_in_subprocess(target, code)
class _UserDefinedTypeFixture:
projects / thirdparty / sqlalchemy / sqlalchemy.git / commitdiff
