From: Mike Bayer <mike_mp@zzzcomputing.com>
Date: Sat, 30 Jan 2021 16:55:22 +0000 (-0500)
Subject: reorganize mapper compile/teardown under registry
X-Git-Tag: rel_1_4_0b2~10^2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5ec5b0a6c7b618bba7926e21f77be9557973860f;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git

reorganize mapper compile/teardown under registry

Mapper "configuration", which occurs within the
:func:`_orm.configure_mappers` function, is now organized to be on a
per-registry basis. This allows for example the mappers within a certain
declarative base to be configured, but not those of another base that is
also present in memory. The goal is to provide a means of reducing
application startup time by only running the "configure" process for sets
of mappers that are needed. This also adds the
:meth:`_orm.registry.configure` method that will run configure for the
mappers local in a particular registry only.

Fixes: #5897
Change-Id: I14bd96982d6d46e241bd6baa2cf97471d21e7caa
---

diff --git a/doc/build/changelog/unreleased_14/5897.rst b/doc/build/changelog/unreleased_14/5897.rst
new file mode 100644
index 0000000000..6a9a11c990
--- /dev/null
+++ b/doc/build/changelog/unreleased_14/5897.rst
@@ -0,0 +1,13 @@
+.. change::
+    :tags: changed, orm
+    :tickets: 5897
+
+    Mapper "configuration", which occurs within the
+    :func:`_orm.configure_mappers` function, is now organized to be on a
+    per-registry basis. This allows for example the mappers within a certain
+    declarative base to be configured, but not those of another base that is
+    also present in memory. The goal is to provide a means of reducing
+    application startup time by only running the "configure" process for sets
+    of mappers that are needed. This also adds the
+    :meth:`_orm.registry.configure` method that will run configure for the
+    mappers local in a particular registry only.
diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py
index f6b1a2e93f..ffbe785032 100644
--- a/lib/sqlalchemy/orm/__init__.py
+++ b/lib/sqlalchemy/orm/__init__.py
@@ -43,7 +43,6 @@ from .interfaces import ONETOMANY  # noqa
 from .interfaces import PropComparator  # noqa
 from .loading import merge_frozen_result  # noqa
 from .loading import merge_result  # noqa
-from .mapper import _mapper_registry
 from .mapper import class_mapper  # noqa
 from .mapper import configure_mappers  # noqa
 from .mapper import Mapper  # noqa
@@ -247,6 +246,10 @@ synonym = public_factory(SynonymProperty, ".orm.synonym")
 def clear_mappers():
     """Remove all mappers from all classes.
 
+    .. versionchanged:: 1.4  This function now locates all
+       :class:`_orm.registry` objects and calls upon the
+       :meth:`_orm.registry.dispose` method of each.
+
     This function removes all instrumentation from classes and disposes
     of their associated mappers.  Once called, the classes are unmapped
     and can be later re-mapped with new mappers.
@@ -266,14 +269,8 @@ def clear_mappers():
     """
 
     with mapperlib._CONFIGURE_MUTEX:
-        while _mapper_registry:
-            try:
-                mapper, b = _mapper_registry.popitem()
-            except KeyError:
-                # weak registry, item could have been collected
-                pass
-            else:
-                mapper.dispose()
+        all_regs = mapperlib._all_registries()
+        mapperlib._dispose_registries(all_regs, False)
 
 
 joinedload = strategy_options.joinedload._unbound_fn
diff --git a/lib/sqlalchemy/orm/base.py b/lib/sqlalchemy/orm/base.py
index b805c6f935..2932f1bb9e 100644
--- a/lib/sqlalchemy/orm/base.py
+++ b/lib/sqlalchemy/orm/base.py
@@ -158,7 +158,6 @@ PASSIVE_ONLY_PERSISTENT = util.symbol(
 
 DEFAULT_MANAGER_ATTR = "_sa_class_manager"
 DEFAULT_STATE_ATTR = "_sa_instance_state"
-_INSTRUMENTOR = ("mapper", "instrumentor")
 
 EXT_CONTINUE = util.symbol("EXT_CONTINUE")
 EXT_STOP = util.symbol("EXT_STOP")
@@ -412,8 +411,8 @@ def _inspect_mapped_class(class_, configure=False):
     except exc.NO_STATE:
         return None
     else:
-        if configure and mapper._new_mappers:
-            mapper._configure_all()
+        if configure:
+            mapper._check_configure()
         return mapper
 
 
diff --git a/lib/sqlalchemy/orm/decl_api.py b/lib/sqlalchemy/orm/decl_api.py
index e6983675bd..e6d41083f1 100644
--- a/lib/sqlalchemy/orm/decl_api.py
+++ b/lib/sqlalchemy/orm/decl_api.py
@@ -7,13 +7,16 @@
 """Public API functions and helpers for declarative."""
 from __future__ import absolute_import
 
+import itertools
 import re
 import weakref
 
 from . import attributes
 from . import clsregistry
 from . import exc as orm_exc
+from . import instrumentation
 from . import interfaces
+from . import mapper as mapperlib
 from .base import _inspect_mapped_class
 from .decl_base import _add_attribute
 from .decl_base import _as_declarative
@@ -456,12 +459,179 @@ class registry(object):
             class_registry = weakref.WeakValueDictionary()
 
         self._class_registry = class_registry
+        self._managers = weakref.WeakKeyDictionary()
+        self._non_primary_mappers = weakref.WeakKeyDictionary()
         self.metadata = lcl_metadata
         self.constructor = constructor
 
+        self._dependents = set()
+        self._dependencies = set()
+
+        self._new_mappers = False
+
+        mapperlib._mapper_registries[self] = True
+
+    @property
+    def mappers(self):
+        """read only collection of all :class:`_orm.Mapper` objects."""
+
+        return frozenset(manager.mapper for manager in self._managers).union(
+            self._non_primary_mappers
+        )
+
+    def _set_depends_on(self, registry):
+        if registry is self:
+            return
+        registry._dependents.add(self)
+        self._dependencies.add(registry)
+
+    def _flag_new_mapper(self, mapper):
+        mapper._ready_for_configure = True
+        if self._new_mappers:
+            return
+
+        for reg in self._recurse_with_dependents({self}):
+            reg._new_mappers = True
+
+    @classmethod
+    def _recurse_with_dependents(cls, registries):
+        todo = registries
+        done = set()
+        while todo:
+            reg = todo.pop()
+            done.add(reg)
+
+            # if yielding would remove dependents, make sure we have
+            # them before
+            todo.update(reg._dependents.difference(done))
+            yield reg
+
+            # if yielding would add dependents, make sure we have them
+            # after
+            todo.update(reg._dependents.difference(done))
+
+    @classmethod
+    def _recurse_with_dependencies(cls, registries):
+        todo = registries
+        done = set()
+        while todo:
+            reg = todo.pop()
+            done.add(reg)
+
+            # if yielding would remove dependencies, make sure we have
+            # them before
+            todo.update(reg._dependencies.difference(done))
+
+            yield reg
+
+            # if yielding would remove dependencies, make sure we have
+            # them before
+            todo.update(reg._dependencies.difference(done))
+
+    def _mappers_to_configure(self):
+        return itertools.chain(
+            (
+                manager.mapper
+                for manager in self._managers
+                if manager.is_mapped
+                and not manager.mapper.configured
+                and manager.mapper._ready_for_configure
+            ),
+            (
+                npm
+                for npm in self._non_primary_mappers
+                if not npm.configured and npm._ready_for_configure
+            ),
+        )
+
+    def _add_non_primary_mapper(self, np_mapper):
+        self._non_primary_mappers[np_mapper] = True
+
     def _dispose_cls(self, cls):
         clsregistry.remove_class(cls.__name__, cls, self._class_registry)
 
+    def _add_manager(self, manager):
+        self._managers[manager] = True
+        assert manager.registry is None
+        manager.registry = self
+
+    def configure(self, cascade=False):
+        """Configure all as-yet unconfigured mappers in this
+        :class:`_orm.registry`.
+
+        The configure step is used to reconcile and initialize the
+        :func:`_orm.relationship` linkages between mapped classes, as well as
+        to invoke configuration events such as the
+        :meth:`_orm.MapperEvents.before_configured` and
+        :meth:`_orm.MapperEvents.after_configured`, which may be used by ORM
+        extensions or user-defined extension hooks.
+
+        If one or more mappers in this registry contain
+        :func:`_orm.relationship` constructs that refer to mapped classes in
+        other registries, this registry is said to be *dependent* on those
+        registries. In order to configure those dependent registries
+        automatically, the :paramref:`_orm.registry.configure.cascade` flag
+        should be set to ``True``. Otherwise, if they are not configured, an
+        exception will be raised.  The rationale behind this behavior is to
+        allow an application to programmatically invoke configuration of
+        registries while controlling whether or not the process implicitly
+        reaches other registries.
+
+        As an alternative to invoking :meth:`_orm.registry.configure`, the ORM
+        function :func:`_orm.configure_mappers` function may be used to ensure
+        configuration is complete for all :class:`_orm.registry` objects in
+        memory. This is generally simpler to use and also predates the usage of
+        :class:`_orm.registry` objects overall. However, this function will
+        impact all mappings throughout the running Python process and may be
+        more memory/time consuming for an application that has many registries
+        in use for different purposes that may not be needed immediately.
+
+        .. seealso::
+
+            :func:`_orm.configure_mappers`
+
+
+        .. versionadded:: 1.4.0b2
+
+        """
+        mapperlib._configure_registries({self}, cascade=cascade)
+
+    def dispose(self, cascade=False):
+        """Dispose of all mappers in this :class:`_orm.registry`.
+
+        After invocation, all the classes that were mapped within this registry
+        will no longer have class instrumentation associated with them. This
+        method is the per-:class:`_orm.registry` analogue to the
+        application-wide :func:`_orm.clear_mappers` function.
+
+        If this registry contains mappers that are dependencies of other
+        registries, typically via :func:`_orm.relationship` links, then those
+        registries must be disposed as well. When such registries exist in
+        relation to this one, their :meth:`_orm.registry.dispose` method will
+        also be called, if the :paramref:`_orm.registry.dispose.cascade` flag
+        is set to ``True``; otherwise, an error is raised if those registries
+        were not already disposed.
+
+        .. versionadded:: 1.4.0b2
+
+        .. seealso::
+
+            :func:`_orm.clear_mappers`
+
+        """
+
+        mapperlib._dispose_registries({self}, cascade=cascade)
+
+    def _dispose_manager_and_mapper(self, manager):
+        if "mapper" in manager.__dict__:
+            mapper = manager.mapper
+
+            mapper._set_dispose_flags()
+
+        class_ = manager.class_
+        self._dispose_cls(class_)
+        instrumentation._instrumentation_factory.unregister(class_)
+
     def generate_base(
         self,
         mapper=None,
@@ -707,6 +877,9 @@ class registry(object):
         return _mapper(self, class_, local_table, kw)
 
 
+mapperlib._legacy_registry = registry()
+
+
 @util.deprecated_params(
     bind=(
         "2.0",
diff --git a/lib/sqlalchemy/orm/instrumentation.py b/lib/sqlalchemy/orm/instrumentation.py
index d2ff72180d..c970bee22a 100644
--- a/lib/sqlalchemy/orm/instrumentation.py
+++ b/lib/sqlalchemy/orm/instrumentation.py
@@ -129,7 +129,7 @@ class ClassManager(HasMemoized, dict):
         if mapper:
             self.mapper = mapper
         if registry:
-            self.registry = registry
+            registry._add_manager(self)
         if declarative_scan:
             self.declarative_scan = declarative_scan
         if expired_attribute_loader:
@@ -278,11 +278,6 @@ class ClassManager(HasMemoized, dict):
 
         setattr(self.class_, self.MANAGER_ATTR, self)
 
-    def dispose(self):
-        """Disassociate this manager from its class."""
-
-        delattr(self.class_, self.MANAGER_ATTR)
-
     @util.hybridmethod
     def manager_getter(self):
         return _default_manager_getter
@@ -359,6 +354,9 @@ class ClassManager(HasMemoized, dict):
             if key in self.local_attrs:
                 self.uninstrument_attribute(key)
 
+        if self.MANAGER_ATTR in self.class_.__dict__:
+            delattr(self.class_, self.MANAGER_ATTR)
+
     def install_descriptor(self, key, inst):
         if key in (self.STATE_ATTR, self.MANAGER_ATTR):
             raise KeyError(
@@ -496,7 +494,7 @@ class _SerializeManager(object):
                 "Python process!" % self.class_,
             )
         elif manager.is_mapped and not manager.mapper.configured:
-            manager.mapper._configure_all()
+            manager.mapper._check_configure()
 
         # setup _sa_instance_state ahead of time so that
         # unpickle events can access the object normally.
@@ -538,10 +536,7 @@ class InstrumentationFactory(object):
     def unregister(self, class_):
         manager = manager_of_class(class_)
         manager.unregister()
-        manager.dispose()
         self.dispatch.class_uninstrument(class_)
-        if ClassManager.MANAGER_ATTR in class_.__dict__:
-            delattr(class_, ClassManager.MANAGER_ATTR)
 
 
 # this attribute is replaced by sqlalchemy.ext.instrumentation
diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py
index bbe39af603..994a76bdc8 100644
--- a/lib/sqlalchemy/orm/interfaces.py
+++ b/lib/sqlalchemy/orm/interfaces.py
@@ -221,6 +221,7 @@ class MapperProperty(
         relationships between mappers and perform other post-mapper-creation
         initialization steps.
 
+
         """
         self._configure_started = True
         self.do_init()
diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py
index 4ad91dd063..66c0627243 100644
--- a/lib/sqlalchemy/orm/mapper.py
+++ b/lib/sqlalchemy/orm/mapper.py
@@ -29,7 +29,6 @@ from . import loading
 from . import properties
 from . import util as orm_util
 from .base import _class_to_mapper
-from .base import _INSTRUMENTOR
 from .base import _state_mapper
 from .base import class_mapper
 from .base import state_str
@@ -57,8 +56,21 @@ from ..sql import visitors
 from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL
 from ..util import HasMemoized
 
+_mapper_registries = weakref.WeakKeyDictionary()
+
+_legacy_registry = None
+
+
+def _all_registries():
+    return set(_mapper_registries)
+
+
+def _unconfigured_mappers():
+    for reg in _mapper_registries:
+        for mapper in reg._mappers_to_configure():
+            yield mapper
+
 
-_mapper_registry = weakref.WeakKeyDictionary()
 _already_compiling = False
 
 
@@ -111,8 +123,8 @@ class Mapper(
 
     """
 
-    _new_mappers = False
     _dispose_called = False
+    _ready_for_configure = False
 
     @util.deprecated_params(
         non_primary=(
@@ -567,7 +579,6 @@ class Mapper(
               techniques.
 
         """
-
         self.class_ = util.assert_arg_type(class_, type, "class_")
         self._sort_key = "%s.%s" % (
             self.class_.__module__,
@@ -620,7 +631,7 @@ class Mapper(
             else None
         )
         self._dependency_processors = []
-        self.validators = util.immutabledict()
+        self.validators = util.EMPTY_DICT
         self.passive_updates = passive_updates
         self.passive_deletes = passive_deletes
         self.legacy_is_orphan = legacy_is_orphan
@@ -662,8 +673,6 @@ class Mapper(
         else:
             self.exclude_properties = None
 
-        self.configured = False
-
         # prevent this mapper from being constructed
         # while a configure_mappers() is occurring (and defer a
         # configure_mappers() until construction succeeds)
@@ -674,7 +683,7 @@ class Mapper(
             self._configure_properties()
             self._configure_polymorphic_setter()
             self._configure_pks()
-            Mapper._new_mappers = True
+            self.registry._flag_new_mapper(self)
             self._log("constructed")
             self._expire_memoizations()
 
@@ -768,7 +777,7 @@ class Mapper(
 
     """
 
-    configured = None
+    configured = False
     """Represent ``True`` if this :class:`_orm.Mapper` has been configured.
 
     This is a *read only* attribute determined during mapper construction.
@@ -1191,8 +1200,9 @@ class Mapper(
                     "Mapper." % self.class_
                 )
             self.class_manager = manager
+            self.registry = manager.registry
             self._identity_class = manager.mapper._identity_class
-            _mapper_registry[self] = True
+            manager.registry._add_non_primary_mapper(self)
             return
 
         if manager is not None:
@@ -1207,8 +1217,6 @@ class Mapper(
             # ClassManager.instrument_attribute() creates
             # new managers for each subclass if they don't yet exist.
 
-        _mapper_registry[self] = True
-
         self.dispatch.instrument_class(self, self.class_)
 
         # this invokes the class_instrument event and sets up
@@ -1227,6 +1235,7 @@ class Mapper(
             # and call the class_instrument event
             finalize=True,
         )
+
         if not manager.registry:
             util.warn_deprecated_20(
                 "Calling the mapper() function directly outside of a "
@@ -1234,18 +1243,17 @@ class Mapper(
                 " Please use the sqlalchemy.orm.registry.map_imperatively() "
                 "function for a classical mapping."
             )
-            from . import registry
-
-            manager.registry = registry()
+            assert _legacy_registry is not None
+            _legacy_registry._add_manager(manager)
 
         self.class_manager = manager
+        self.registry = manager.registry
 
         # The remaining members can be added by any mapper,
         # e_name None or not.
-        if manager.info.get(_INSTRUMENTOR, False):
+        if manager.mapper is None:
             return
 
-        event.listen(manager, "first_init", _event_on_first_init, raw=True)
         event.listen(manager, "init", _event_on_init, raw=True)
 
         for key, method in util.iterate_attributes(self.class_):
@@ -1270,29 +1278,12 @@ class Mapper(
                             {name: (method, validation_opts)}
                         )
 
-        manager.info[_INSTRUMENTOR] = self
-
-    @classmethod
-    def _configure_all(cls):
-        """Class-level path to the :func:`.configure_mappers` call."""
-        configure_mappers()
-
-    def dispose(self):
-        # Disable any attribute-based compilation.
+    def _set_dispose_flags(self):
         self.configured = True
+        self._ready_for_configure = True
         self._dispose_called = True
 
-        if hasattr(self, "_configure_failed"):
-            del self._configure_failed
-
-        if (
-            not self.non_primary
-            and self.class_manager is not None
-            and self.class_manager.is_mapped
-            and self.class_manager.mapper is self
-        ):
-            self.class_manager.registry._dispose_cls(self.class_)
-            instrumentation.unregister_class(self.class_)
+        self.__dict__.pop("_configure_failed", None)
 
     def _configure_pks(self):
         self.tables = sql_util.find_tables(self.persist_selectable)
@@ -1877,6 +1868,10 @@ class Mapper(
                 "columns get mapped." % (key, self, column.key, prop)
             )
 
+    def _check_configure(self):
+        if self.registry._new_mappers:
+            _configure_registries({self.registry}, cascade=True)
+
     def _post_configure_properties(self):
         """Call the ``init()`` method on all ``MapperProperties``
         attached to this mapper.
@@ -1983,8 +1978,8 @@ class Mapper(
     def get_property(self, key, _configure_mappers=True):
         """return a MapperProperty associated with the given key."""
 
-        if _configure_mappers and Mapper._new_mappers:
-            configure_mappers()
+        if _configure_mappers:
+            self._check_configure()
 
         try:
             return self._props[key]
@@ -2005,8 +2000,8 @@ class Mapper(
     @property
     def iterate_properties(self):
         """return an iterator of all MapperProperty objects."""
-        if Mapper._new_mappers:
-            configure_mappers()
+
+        self._check_configure()
         return iter(self._props.values())
 
     def _mappers_from_spec(self, spec, selectable):
@@ -2081,8 +2076,8 @@ class Mapper(
 
     @HasMemoized.memoized_attribute
     def _with_polymorphic_mappers(self):
-        if Mapper._new_mappers:
-            configure_mappers()
+        self._check_configure()
+
         if not self.with_polymorphic:
             return []
         return self._mappers_from_spec(*self.with_polymorphic)
@@ -2098,8 +2093,7 @@ class Mapper(
         This allows the inspection process run a configure mappers hook.
 
         """
-        if Mapper._new_mappers:
-            configure_mappers()
+        self._check_configure()
 
     @HasMemoized.memoized_attribute
     def _with_polymorphic_selectable(self):
@@ -2403,8 +2397,8 @@ class Mapper(
             :attr:`_orm.Mapper.all_orm_descriptors`
 
         """
-        if Mapper._new_mappers:
-            configure_mappers()
+
+        self._check_configure()
         return util.ImmutableProperties(self._props)
 
     @HasMemoized.memoized_attribute
@@ -2559,8 +2553,7 @@ class Mapper(
         )
 
     def _filter_properties(self, type_):
-        if Mapper._new_mappers:
-            configure_mappers()
+        self._check_configure()
         return util.ImmutableProperties(
             util.OrderedDict(
                 (k, v) for k, v in self._props.items() if isinstance(v, type_)
@@ -3288,24 +3281,54 @@ class _OptGetColumnsNotAvailable(Exception):
 
 def configure_mappers():
     """Initialize the inter-mapper relationships of all mappers that
-    have been constructed thus far.
-
-    This function can be called any number of times, but in
-    most cases is invoked automatically, the first time mappings are used,
-    as well as whenever mappings are used and additional not-yet-configured
-    mappers have been constructed.
-
-    Points at which this occur include when a mapped class is instantiated
-    into an instance, as well as when the :meth:`.Session.query` method
-    is used.
-
-    The :func:`.configure_mappers` function provides several event hooks
-    that can be used to augment its functionality.  These methods include:
+    have been constructed thus far across all :class:`_orm.registry`
+    collections.
+
+    The configure step is used to reconcile and initialize the
+    :func:`_orm.relationship` linkages between mapped classes, as well as to
+    invoke configuration events such as the
+    :meth:`_orm.MapperEvents.before_configured` and
+    :meth:`_orm.MapperEvents.after_configured`, which may be used by ORM
+    extensions or user-defined extension hooks.
+
+    Mapper configuration is normally invoked automatically, the first time
+    mappings from a particular :class:`_orm.registry` are used, as well as
+    whenever mappings are used and additional not-yet-configured mappers have
+    been constructed. The automatic configuration process however is local only
+    to the :class:`_orm.registry` involving the target mapper and any related
+    :class:`_orm.registry` objects which it may depend on; this is
+    equivalent to invoking the :meth:`_orm.registry.configure` method
+    on a particular :class:`_orm.registry`.
+
+    By contrast, the :func:`_orm.configure_mappers` function will invoke the
+    configuration process on all :class:`_orm.registry` objects that
+    exist in memory, and may be useful for scenarios where many individual
+    :class:`_orm.registry` objects that are nonetheless interrelated are
+    in use.
+
+    .. versionchanged:: 1.4
+
+        As of SQLAlchemy 1.4.0b2, this function works on a
+        per-:class:`_orm.registry` basis, locating all :class:`_orm.registry`
+        objects present and invoking the :meth:`_orm.registry.configure` method
+        on each. The :meth:`_orm.registry.configure` method may be preferred to
+        limit the configuration of mappers to those local to a particular
+        :class:`_orm.registry` and/or declarative base class.
+
+    Points at which automatic configuration is invoked include when a mapped
+    class is instantiated into an instance, as well as when ORM queries
+    are emitted using :meth:`.Session.query` or :meth:`_orm.Session.execute`
+    with an ORM-enabled statement.
+
+    The mapper configure process, whether invoked by
+    :func:`_orm.configure_mappers` or from :meth:`_orm.registry.configure`,
+    provides several event hooks that can be used to augment the mapper
+    configuration step. These hooks include:
 
     * :meth:`.MapperEvents.before_configured` - called once before
-      :func:`.configure_mappers` does any work; this can be used to establish
-      additional options, properties, or related mappings before the operation
-      proceeds.
+      :func:`.configure_mappers` or :meth:`_orm.registry.configure` does any
+      work; this can be used to establish additional options, properties, or
+      related mappings before the operation proceeds.
 
     * :meth:`.MapperEvents.mapper_configured` - called as each individual
       :class:`_orm.Mapper` is configured within the process; will include all
@@ -3313,15 +3336,25 @@ def configure_mappers():
       to be configured.
 
     * :meth:`.MapperEvents.after_configured` - called once after
-      :func:`.configure_mappers` is complete; at this stage, all
-      :class:`_orm.Mapper` objects that are known  to SQLAlchemy will be fully
-      configured.  Note that the calling application may still have other
-      mappings that haven't been produced yet, such as if they are in modules
-      as yet unimported.
+      :func:`.configure_mappers` or :meth:`_orm.registry.configure` is
+      complete; at this stage, all :class:`_orm.Mapper` objects that fall
+      within the scope of the configuration operation will be fully configured.
+      Note that the calling application may still have other mappings that
+      haven't been produced yet, such as if they are in modules as yet
+      unimported, and may also have mappings that are still to be configured,
+      if they are in other :class:`_orm.registry` collections not part of the
+      current scope of configuration.
 
     """
 
-    if not Mapper._new_mappers:
+    _configure_registries(set(_mapper_registries), cascade=True)
+
+
+def _configure_registries(registries, cascade):
+    for reg in registries:
+        if reg._new_mappers:
+            break
+    else:
         return
 
     with _CONFIGURE_MUTEX:
@@ -3332,58 +3365,105 @@ def configure_mappers():
         try:
 
             # double-check inside mutex
-            if not Mapper._new_mappers:
+            for reg in registries:
+                if reg._new_mappers:
+                    break
+            else:
                 return
 
-            has_skip = False
-
             Mapper.dispatch._for_class(Mapper).before_configured()
             # initialize properties on all mappers
             # note that _mapper_registry is unordered, which
             # may randomly conceal/reveal issues related to
             # the order of mapper compilation
 
-            for mapper in list(_mapper_registry):
-                run_configure = None
-                for fn in mapper.dispatch.before_mapper_configured:
-                    run_configure = fn(mapper, mapper.class_)
-                    if run_configure is EXT_SKIP:
-                        has_skip = True
-                        break
-                if run_configure is EXT_SKIP:
-                    continue
-
-                if getattr(mapper, "_configure_failed", False):
-                    e = sa_exc.InvalidRequestError(
-                        "One or more mappers failed to initialize - "
-                        "can't proceed with initialization of other "
-                        "mappers. Triggering mapper: '%s'. "
-                        "Original exception was: %s"
-                        % (mapper, mapper._configure_failed)
-                    )
-                    e._configure_failed = mapper._configure_failed
-                    raise e
-
-                if not mapper.configured:
-                    try:
-                        mapper._post_configure_properties()
-                        mapper._expire_memoizations()
-                        mapper.dispatch.mapper_configured(
-                            mapper, mapper.class_
-                        )
-                    except Exception:
-                        exc = sys.exc_info()[1]
-                        if not hasattr(exc, "_configure_failed"):
-                            mapper._configure_failed = exc
-                        raise
-
-            if not has_skip:
-                Mapper._new_mappers = False
+            _do_configure_registries(registries, cascade)
         finally:
             _already_compiling = False
     Mapper.dispatch._for_class(Mapper).after_configured()
 
 
+@util.preload_module("sqlalchemy.orm.decl_api")
+def _do_configure_registries(registries, cascade):
+
+    registry = util.preloaded.orm_decl_api.registry
+
+    orig = set(registries)
+
+    for reg in registry._recurse_with_dependencies(registries):
+        has_skip = False
+
+        for mapper in reg._mappers_to_configure():
+            run_configure = None
+            for fn in mapper.dispatch.before_mapper_configured:
+                run_configure = fn(mapper, mapper.class_)
+                if run_configure is EXT_SKIP:
+                    has_skip = True
+                    break
+            if run_configure is EXT_SKIP:
+                continue
+
+            if getattr(mapper, "_configure_failed", False):
+                e = sa_exc.InvalidRequestError(
+                    "One or more mappers failed to initialize - "
+                    "can't proceed with initialization of other "
+                    "mappers. Triggering mapper: '%s'. "
+                    "Original exception was: %s"
+                    % (mapper, mapper._configure_failed)
+                )
+                e._configure_failed = mapper._configure_failed
+                raise e
+
+            if not mapper.configured:
+                try:
+                    mapper._post_configure_properties()
+                    mapper._expire_memoizations()
+                    mapper.dispatch.mapper_configured(mapper, mapper.class_)
+                except Exception:
+                    exc = sys.exc_info()[1]
+                    if not hasattr(exc, "_configure_failed"):
+                        mapper._configure_failed = exc
+                    raise
+        if not has_skip:
+            reg._new_mappers = False
+
+        if not cascade and reg._dependencies.difference(orig):
+            raise sa_exc.InvalidRequestError(
+                "configure was called with cascade=False but "
+                "additional registries remain"
+            )
+
+
+@util.preload_module("sqlalchemy.orm.decl_api")
+def _dispose_registries(registries, cascade):
+
+    registry = util.preloaded.orm_decl_api.registry
+
+    orig = set(registries)
+
+    for reg in registry._recurse_with_dependents(registries):
+        if not cascade and reg._dependents.difference(orig):
+            raise sa_exc.InvalidRequestError(
+                "Registry has dependent registries that are not disposed; "
+                "pass cascade=True to clear these also"
+            )
+
+        while reg._managers:
+            manager, _ = reg._managers.popitem()
+            reg._dispose_manager_and_mapper(manager)
+
+        reg._non_primary_mappers.clear()
+        reg._dependents.clear()
+        for dep in reg._dependencies:
+            dep._dependents.discard(reg)
+        reg._dependencies.clear()
+        # this wasn't done in the 1.3 clear_mappers() and in fact it
+        # was a bug, as it could cause configure_mappers() to invoke
+        # the "before_configured" event even though mappers had all been
+        # disposed.
+        reg._new_mappers = False
+
+
 def reconstructor(fn):
     """Decorate a method as the 'reconstructor' hook.
 
@@ -3460,25 +3540,12 @@ def validates(*names, **kw):
 
 
 def _event_on_load(state, ctx):
-    instrumenting_mapper = state.manager.info[_INSTRUMENTOR]
+    instrumenting_mapper = state.manager.mapper
+
     if instrumenting_mapper._reconstructor:
         instrumenting_mapper._reconstructor(state.obj())
 
 
-def _event_on_first_init(manager, cls):
-    """Initial mapper compilation trigger.
-
-    instrumentation calls this one when InstanceState
-    is first generated, and is needed for legacy mutable
-    attributes to work.
-    """
-
-    instrumenting_mapper = manager.info.get(_INSTRUMENTOR)
-    if instrumenting_mapper:
-        if Mapper._new_mappers:
-            configure_mappers()
-
-
 def _event_on_init(state, args, kwargs):
     """Run init_instance hooks.
 
@@ -3488,10 +3555,9 @@ def _event_on_init(state, args, kwargs):
 
     """
 
-    instrumenting_mapper = state.manager.info.get(_INSTRUMENTOR)
+    instrumenting_mapper = state.manager.mapper
     if instrumenting_mapper:
-        if Mapper._new_mappers:
-            configure_mappers()
+        instrumenting_mapper._check_configure()
         if instrumenting_mapper._set_polymorphic_identity:
             instrumenting_mapper._set_polymorphic_identity(state)
 
diff --git a/lib/sqlalchemy/orm/relationships.py b/lib/sqlalchemy/orm/relationships.py
index 41c3a5e53c..1a01409148 100644
--- a/lib/sqlalchemy/orm/relationships.py
+++ b/lib/sqlalchemy/orm/relationships.py
@@ -1658,11 +1658,8 @@ class RelationshipProperty(StrategizedProperty):
                 return _orm_annotate(self.__negated_contains_or_equals(other))
 
         @util.memoized_property
-        @util.preload_module("sqlalchemy.orm.mapper")
         def property(self):
-            mapperlib = util.preloaded.orm_mapper
-            if mapperlib.Mapper._new_mappers:
-                mapperlib.Mapper._configure_all()
+            self.prop.parent._check_configure()
             return self.prop
 
     def _with_parent(self, instance, alias_secondary=True, from_entity=None):
@@ -2130,9 +2127,9 @@ class RelationshipProperty(StrategizedProperty):
         return self.entity.mapper
 
     def do_init(self):
-
         self._check_conflicts()
         self._process_dependent_arguments()
+        self._setup_registry_dependencies()
         self._setup_join_conditions()
         self._check_cascade_settings(self._cascade)
         self._post_init()
@@ -2141,6 +2138,11 @@ class RelationshipProperty(StrategizedProperty):
         super(RelationshipProperty, self).do_init()
         self._lazy_strategy = self._get_strategy((("lazy", "select"),))
 
+    def _setup_registry_dependencies(self):
+        self.parent.mapper.registry._set_depends_on(
+            self.entity.mapper.registry
+        )
+
     def _process_dependent_arguments(self):
         """Convert incoming configuration arguments to their
         proper form.
@@ -3391,9 +3393,7 @@ class JoinCondition(object):
 
     _track_overlapping_sync_targets = weakref.WeakKeyDictionary()
 
-    @util.preload_module("sqlalchemy.orm.mapper")
     def _warn_for_conflicting_sync_targets(self):
-        mapperlib = util.preloaded.orm_mapper
         if not self.support_sync:
             return
 
@@ -3424,7 +3424,7 @@ class JoinCondition(object):
 
                 for pr, fr_ in prop_to_from.items():
                     if (
-                        pr.mapper in mapperlib._mapper_registry
+                        not pr.mapper._dispose_called
                         and pr not in self.prop._reverse_property
                         and pr.key not in self.prop._overlaps
                         and self.prop.key not in pr._overlaps
diff --git a/test/aaa_profiling/test_memusage.py b/test/aaa_profiling/test_memusage.py
index a41a8b9f11..dd709965ba 100644
--- a/test/aaa_profiling/test_memusage.py
+++ b/test/aaa_profiling/test_memusage.py
@@ -27,7 +27,6 @@ from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm import subqueryload
-from sqlalchemy.orm.mapper import _mapper_registry
 from sqlalchemy.orm.session import _sessions
 from sqlalchemy.processors import to_decimal_processor_factory
 from sqlalchemy.processors import to_unicode_processor_factory
@@ -237,13 +236,12 @@ def profile_memory(
 def assert_no_mappers():
     clear_mappers()
     gc_collect()
-    assert len(_mapper_registry) == 0
 
 
 class EnsureZeroed(fixtures.ORMTest):
     def setup_test(self):
         _sessions.clear()
-        _mapper_registry.clear()
+        clear_mappers()
 
         # enable query caching, however make the cache small so that
         # the tests don't take too long.  issues w/ caching include making
diff --git a/test/ext/declarative/test_inheritance.py b/test/ext/declarative/test_inheritance.py
index e25e7cfc29..496b5579d9 100644
--- a/test/ext/declarative/test_inheritance.py
+++ b/test/ext/declarative/test_inheritance.py
@@ -227,7 +227,7 @@ class ConcreteInhTest(
             Employee,
         )
 
-        configure_mappers()
+        Base.registry.configure()
 
         # no subclasses yet.
         assert_raises_message(
@@ -257,7 +257,7 @@ class ConcreteInhTest(
             Employee,
         )
 
-        configure_mappers()
+        Base.registry.configure()
 
         self.assert_compile(
             Session().query(Employee),
diff --git a/test/orm/declarative/test_basic.py b/test/orm/declarative/test_basic.py
index 4d9162105b..c779d214c8 100644
--- a/test/orm/declarative/test_basic.py
+++ b/test/orm/declarative/test_basic.py
@@ -547,8 +547,6 @@ class DeclarativeTest(DeclarativeTestBase):
     def test_recompile_on_othermapper(self):
         """declarative version of the same test in mappers.py"""
 
-        from sqlalchemy.orm import mapperlib
-
         class User(Base):
             __tablename__ = "users"
 
@@ -565,10 +563,10 @@ class DeclarativeTest(DeclarativeTestBase):
                 "User", primaryjoin=user_id == User.id, backref="addresses"
             )
 
-        assert mapperlib.Mapper._new_mappers is True
+        assert User.__mapper__.registry._new_mappers is True
         u = User()  # noqa
         assert User.addresses
-        assert mapperlib.Mapper._new_mappers is False
+        assert User.__mapper__.registry._new_mappers is False
 
     def test_string_dependency_resolution(self):
         class User(Base, fixtures.ComparableEntity):
diff --git a/test/orm/test_events.py b/test/orm/test_events.py
index d9caa221d6..6ba94096d1 100644
--- a/test/orm/test_events.py
+++ b/test/orm/test_events.py
@@ -19,13 +19,13 @@ from sqlalchemy.orm import EXT_SKIP
 from sqlalchemy.orm import instrumentation
 from sqlalchemy.orm import Mapper
 from sqlalchemy.orm import mapper
+from sqlalchemy.orm import mapperlib
 from sqlalchemy.orm import query
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm import subqueryload
-from sqlalchemy.orm.mapper import _mapper_registry
 from sqlalchemy.testing import assert_raises
 from sqlalchemy.testing import assert_raises_message
 from sqlalchemy.testing import AssertsCompiledSQL
@@ -1073,7 +1073,11 @@ class MapperEventsTest(_RemoveListeners, _fixtures.FixtureTest):
             canary.mock_calls,
         )
 
-    def test_before_mapper_configured_event(self):
+    @testing.combinations((True,), (False,), argnames="create_dependency")
+    @testing.combinations((True,), (False,), argnames="configure_at_once")
+    def test_before_mapper_configured_event(
+        self, create_dependency, configure_at_once
+    ):
         """Test [ticket:4397].
 
         This event is intended to allow a specific mapper to be skipped during
@@ -1088,7 +1092,7 @@ class MapperEventsTest(_RemoveListeners, _fixtures.FixtureTest):
         """
 
         User, users = self.classes.User, self.tables.users
-        mapper(User, users)
+        ump = mapper(User, users)
 
         AnotherBase = declarative_base()
 
@@ -1098,12 +1102,18 @@ class MapperEventsTest(_RemoveListeners, _fixtures.FixtureTest):
             __mapper_args__ = dict(
                 polymorphic_on="species", polymorphic_identity="Animal"
             )
+            if create_dependency:
+                user_id = Column("user_id", ForeignKey(users.c.id))
 
-        # Register the first classes and create their Mappers:
-        configure_mappers()
+        if not configure_at_once:
+            # Register the first classes and create their Mappers:
+            configure_mappers()
+
+            unconfigured = list(mapperlib._unconfigured_mappers())
+            eq_(0, len(unconfigured))
 
-        unconfigured = [m for m in _mapper_registry if not m.configured]
-        eq_(0, len(unconfigured))
+        if create_dependency:
+            ump.add_property("animal", relationship(Animal))
 
         # Declare a subclass, table and mapper, which refers to one that has
         # not been loaded yet (Employer), and therefore cannot be configured:
@@ -1111,8 +1121,12 @@ class MapperEventsTest(_RemoveListeners, _fixtures.FixtureTest):
             nonexistent = relationship("Nonexistent")
 
         # These new classes should not be configured at this point:
-        unconfigured = [m for m in _mapper_registry if not m.configured]
-        eq_(1, len(unconfigured))
+        unconfigured = list(mapperlib._unconfigured_mappers())
+
+        if configure_at_once:
+            eq_(3, len(unconfigured))
+        else:
+            eq_(1, len(unconfigured))
 
         # Now try to query User, which is internally consistent. This query
         # fails by default because Mammal needs to be configured, and cannot
@@ -1121,7 +1135,10 @@ class MapperEventsTest(_RemoveListeners, _fixtures.FixtureTest):
             s = fixture_session()
             s.query(User)
 
-        assert_raises(sa.exc.InvalidRequestError, probe)
+        if create_dependency:
+            assert_raises(sa.exc.InvalidRequestError, probe)
+        else:
+            probe()
 
         # If we disable configuring mappers while querying, then it succeeds:
         @event.listens_for(
diff --git a/test/orm/test_mapper.py b/test/orm/test_mapper.py
index d182fd2c17..ba4cf26373 100644
--- a/test/orm/test_mapper.py
+++ b/test/orm/test_mapper.py
@@ -15,6 +15,7 @@ from sqlalchemy.orm import aliased
 from sqlalchemy.orm import attributes
 from sqlalchemy.orm import backref
 from sqlalchemy.orm import class_mapper
+from sqlalchemy.orm import clear_mappers
 from sqlalchemy.orm import column_property
 from sqlalchemy.orm import composite
 from sqlalchemy.orm import configure_mappers
@@ -31,8 +32,11 @@ from sqlalchemy.testing import assert_raises
 from sqlalchemy.testing import assert_raises_message
 from sqlalchemy.testing import AssertsCompiledSQL
 from sqlalchemy.testing import eq_
+from sqlalchemy.testing import expect_raises_message
 from sqlalchemy.testing import fixtures
 from sqlalchemy.testing import is_
+from sqlalchemy.testing import is_false
+from sqlalchemy.testing import is_true
 from sqlalchemy.testing import ne_
 from sqlalchemy.testing.fixtures import ComparableMixin
 from sqlalchemy.testing.fixtures import fixture_session
@@ -304,9 +308,9 @@ class MapperTest(_fixtures.FixtureTest, AssertsCompiledSQL):
             self.classes.User,
         )
 
-        self.mapper(User, users)
+        mp = self.mapper(User, users)
         sa.orm.configure_mappers()
-        assert sa.orm.mapperlib.Mapper._new_mappers is False
+        assert mp.registry._new_mappers is False
 
         m = self.mapper(
             Address,
@@ -315,10 +319,10 @@ class MapperTest(_fixtures.FixtureTest, AssertsCompiledSQL):
         )
 
         assert m.configured is False
-        assert sa.orm.mapperlib.Mapper._new_mappers is True
+        assert m.registry._new_mappers is True
         User()
         assert User.addresses
-        assert sa.orm.mapperlib.Mapper._new_mappers is False
+        assert m.registry._new_mappers is False
 
     def test_configure_on_session(self):
         User, users = self.classes.User, self.tables.users
@@ -1810,6 +1814,35 @@ class MapperTest(_fixtures.FixtureTest, AssertsCompiledSQL):
         self.mapper(Address, addresses)
         configure_mappers()
 
+    @testing.combinations((True,), (False,))
+    def test_registry_configure(self, cascade):
+        User, users = self.classes.User, self.tables.users
+
+        reg1 = registry()
+        ump = reg1.map_imperatively(User, users)
+
+        reg2 = registry()
+        AnotherBase = reg2.generate_base()
+
+        class Animal(AnotherBase):
+            __tablename__ = "animal"
+            species = Column(String(30), primary_key=True)
+            __mapper_args__ = dict(
+                polymorphic_on="species", polymorphic_identity="Animal"
+            )
+            user_id = Column("user_id", ForeignKey(users.c.id))
+
+        ump.add_property("animal", relationship(Animal))
+
+        if cascade:
+            reg1.configure(cascade=True)
+        else:
+            with expect_raises_message(
+                sa.exc.InvalidRequestError,
+                "configure was called with cascade=False",
+            ):
+                reg1.configure()
+
     def test_reconstructor(self):
         users = self.tables.users
 
@@ -2810,3 +2843,158 @@ class ComparatorFactoryTest(_fixtures.FixtureTest, AssertsCompiledSQL):
             "foobar(users_1.id) = foobar(:foobar_1)",
             dialect=default.DefaultDialect(),
         )
+
+
+class RegistryConfigDisposeTest(fixtures.TestBase):
+    """test the cascading behavior of registry configure / dispose."""
+
+    @testing.fixture
+    def threeway_fixture(self):
+        reg1 = registry()
+        reg2 = registry()
+        reg3 = registry()
+
+        ab = bc = True
+
+        @reg1.mapped
+        class A(object):
+            __tablename__ = "a"
+            id = Column(Integer, primary_key=True)
+
+        @reg2.mapped
+        class B(object):
+            __tablename__ = "b"
+            id = Column(Integer, primary_key=True)
+            a_id = Column(ForeignKey(A.id))
+
+        @reg3.mapped
+        class C(object):
+            __tablename__ = "c"
+            id = Column(Integer, primary_key=True)
+            b_id = Column(ForeignKey(B.id))
+
+        if ab:
+            A.__mapper__.add_property("b", relationship(B))
+
+        if bc:
+            B.__mapper__.add_property("c", relationship(C))
+
+        yield reg1, reg2, reg3
+
+        clear_mappers()
+
+    @testing.fixture
+    def threeway_configured_fixture(self, threeway_fixture):
+        reg1, reg2, reg3 = threeway_fixture
+        configure_mappers()
+
+        return reg1, reg2, reg3
+
+    @testing.combinations((True,), (False,), argnames="cascade")
+    def test_configure_cascade_on_dependencies(
+        self, threeway_fixture, cascade
+    ):
+        reg1, reg2, reg3 = threeway_fixture
+        A, B, C = (
+            reg1._class_registry["A"],
+            reg2._class_registry["B"],
+            reg3._class_registry["C"],
+        )
+
+        is_(reg3._new_mappers, True)
+        is_(reg2._new_mappers, True)
+        is_(reg1._new_mappers, True)
+
+        if cascade:
+            reg1.configure(cascade=True)
+
+            is_(reg3._new_mappers, False)
+            is_(reg2._new_mappers, False)
+            is_(reg1._new_mappers, False)
+
+            is_true(C.__mapper__.configured)
+            is_true(B.__mapper__.configured)
+            is_true(A.__mapper__.configured)
+        else:
+            with testing.expect_raises_message(
+                sa.exc.InvalidRequestError,
+                "configure was called with cascade=False but additional ",
+            ):
+                reg1.configure()
+
+    @testing.combinations((True,), (False,), argnames="cascade")
+    def test_configure_cascade_not_on_dependents(
+        self, threeway_fixture, cascade
+    ):
+        reg1, reg2, reg3 = threeway_fixture
+        A, B, C = (
+            reg1._class_registry["A"],
+            reg2._class_registry["B"],
+            reg3._class_registry["C"],
+        )
+
+        is_(reg3._new_mappers, True)
+        is_(reg2._new_mappers, True)
+        is_(reg1._new_mappers, True)
+
+        reg3.configure(cascade=cascade)
+
+        is_(reg3._new_mappers, False)
+        is_(reg2._new_mappers, True)
+        is_(reg1._new_mappers, True)
+
+        is_true(C.__mapper__.configured)
+        is_false(B.__mapper__.configured)
+        is_false(A.__mapper__.configured)
+
+    @testing.combinations((True,), (False,), argnames="cascade")
+    def test_dispose_cascade_not_on_dependencies(
+        self, threeway_configured_fixture, cascade
+    ):
+        reg1, reg2, reg3 = threeway_configured_fixture
+        A, B, C = (
+            reg1._class_registry["A"],
+            reg2._class_registry["B"],
+            reg3._class_registry["C"],
+        )
+        am, bm, cm = A.__mapper__, B.__mapper__, C.__mapper__
+
+        reg1.dispose(cascade=cascade)
+
+        eq_(reg3.mappers, {cm})
+        eq_(reg2.mappers, {bm})
+        eq_(reg1.mappers, set())
+
+        is_false(cm._dispose_called)
+        is_false(bm._dispose_called)
+        is_true(am._dispose_called)
+
+    @testing.combinations((True,), (False,), argnames="cascade")
+    def test_clear_cascade_not_on_dependents(
+        self, threeway_configured_fixture, cascade
+    ):
+        reg1, reg2, reg3 = threeway_configured_fixture
+        A, B, C = (
+            reg1._class_registry["A"],
+            reg2._class_registry["B"],
+            reg3._class_registry["C"],
+        )
+        am, bm, cm = A.__mapper__, B.__mapper__, C.__mapper__
+
+        if cascade:
+            reg3.dispose(cascade=True)
+
+            eq_(reg3.mappers, set())
+            eq_(reg2.mappers, set())
+            eq_(reg1.mappers, set())
+
+            is_true(cm._dispose_called)
+            is_true(bm._dispose_called)
+            is_true(am._dispose_called)
+        else:
+            with testing.expect_raises_message(
+                sa.exc.InvalidRequestError,
+                "Registry has dependent registries that are not disposed; "
+                "pass cascade=True to clear these also",
+            ):
+                reg3.dispose()