gh-86298: Ensure that __loader__ and __spec__.loader agree in warnings.warn_explicit...

In `_warnings.c`, in the C equivalent of `warnings.warn_explicit()`, if the module globals are given (and not None), the warning will attempt to get the source line for the issued warning.  To do this, it needs the module's loader.

Previously, it would only look up `__loader__` in the module globals.  In https://github.com/python/cpython/issues/86298 we want to defer to the `__spec__.loader` if available.

The first step on this journey is to check that `loader == __spec__.loader` and issue another warning if it is not.  This commit does that.

Since this is a PoC, only manual testing for now.

```python
# /tmp/foo.py
import warnings

import bar

warnings.warn_explicit(
    'warning!',
    RuntimeWarning,
    'bar.py', 2,
    module='bar knee',
    module_globals=bar.__dict__,
    )
```

```python
# /tmp/bar.py
import sys
import os
import pathlib

# __loader__ = pathlib.Path()
```

Then running this: `./python.exe -Wdefault /tmp/foo.py`

Produces:

```
bar.py:2: RuntimeWarning: warning!
  import os
```

Uncomment the `__loader__ = ` line in `bar.py` and try it again:

```
sys:1: ImportWarning: Module bar; __loader__ != __spec__.loader (<_frozen_importlib_external.SourceFileLoader object at 0x109f7dfa0> != PosixPath('.'))
bar.py:2: RuntimeWarning: warning!
  import os
```

Automerge-Triggered-By: GH:warsaw

diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
index 3fa875f52e013ef2963785c89b92bc2156ac37b9..b22b5251f1de469b4e3d1413af47192094a69a71 100644 (file)
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -558,6 +558,11 @@ listed below.
    It is **strongly** recommended that you rely on :attr:`__spec__`
    instead instead of this attribute.
 
+   .. versionchanged:: 3.12
+      The value of ``__loader__`` is expected to be the same as
+      ``__spec__.loader``.  The use of ``__loader__`` is deprecated and slated
+      for removal in Python 3.14.
+
 .. attribute:: __package__
 
    The module's ``__package__`` attribute may be set.  Its value must
@@ -568,6 +573,9 @@ listed below.
    submodules, to the parent package's name.  See :pep:`366` for further
    details.
 
+   This attribute is used instead of ``__name__`` to calculate explicit
+   relative imports for main modules, as defined in :pep:`366`.
+
    It is **strongly** recommended that you rely on :attr:`__spec__`
    instead instead of this attribute.
 

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 39d63ae9d6e98c7526340de6d21bdfe76b5dabca..fc50e70539cacfc16b2b2bdfa018a269ba2e38a0 100644 (file)
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -862,6 +862,54 @@ def spec_from_file_location(name, location=None, *, loader=None,
     return spec
 
 
+def _bless_my_loader(module_globals):
+    """Helper function for _warnings.c
+
+    See GH#97850 for details.
+    """
+    # 2022-10-06(warsaw): For now, this helper is only used in _warnings.c and
+    # that use case only has the module globals.  This function could be
+    # extended to accept either that or a module object.  However, in the
+    # latter case, it would be better to raise certain exceptions when looking
+    # at a module, which should have either a __loader__ or __spec__.loader.
+    # For backward compatibility, it is possible that we'll get an empty
+    # dictionary for the module globals, and that cannot raise an exception.
+    if not isinstance(module_globals, dict):
+        return None
+
+    missing = object()
+    loader = module_globals.get('__loader__', None)
+    spec = module_globals.get('__spec__', missing)
+
+    if loader is None:
+        if spec is missing:
+            # If working with a module:
+            # raise AttributeError('Module globals is missing a __spec__')
+            return None
+        elif spec is None:
+            raise ValueError('Module globals is missing a __spec__.loader')
+
+    spec_loader = getattr(spec, 'loader', missing)
+
+    if spec_loader in (missing, None):
+        if loader is None:
+            exc = AttributeError if spec_loader is missing else ValueError
+            raise exc('Module globals is missing a __spec__.loader')
+        _warnings.warn(
+            'Module globals is missing a __spec__.loader',
+            DeprecationWarning)
+        spec_loader = loader
+
+    assert spec_loader is not None
+    if loader is not None and loader != spec_loader:
+        _warnings.warn(
+            'Module globals; __loader__ != __spec__.loader',
+            DeprecationWarning)
+        return loader
+
+    return spec_loader
+
+
 # Loaders #####################################################################
 
 class WindowsRegistryFinder:

diff --git a/Lib/test/test_importlib/import_/test_helpers.py b/Lib/test/test_importlib/import_/test_helpers.py
index 90df56f09fe52d096c9af33f26dd95e822227166..550f88d1d7a6516691fe2a583a9f1ec4ad94a256 100644 (file)
--- a/Lib/test/test_importlib/import_/test_helpers.py
+++ b/Lib/test/test_importlib/import_/test_helpers.py
@@ -2,7 +2,9 @@
 
 from importlib import _bootstrap_external, machinery
 import os.path
+from types import ModuleType, SimpleNamespace
 import unittest
+import warnings
 
 from .. import util
 
@@ -67,5 +69,116 @@ class FixUpModuleTests:
 
 FrozenFixUpModuleTests, SourceFixUpModuleTests = util.test_both(FixUpModuleTests)
 
+
+class TestBlessMyLoader(unittest.TestCase):
+    # GH#86298 is part of the migration away from module attributes and toward
+    # __spec__ attributes.  There are several cases to test here.  This will
+    # have to change in Python 3.14 when we actually remove/ignore __loader__
+    # in favor of requiring __spec__.loader.
+
+    def test_gh86298_no_loader_and_no_spec(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        del bar.__spec__
+        # 2022-10-06(warsaw): For backward compatibility with the
+        # implementation in _warnings.c, this can't raise an
+        # AttributeError.  See _bless_my_loader() in _bootstrap_external.py
+        # If working with a module:
+        ## self.assertRaises(
+        ##     AttributeError, _bootstrap_external._bless_my_loader,
+        ##     bar.__dict__)
+        self.assertIsNone(_bootstrap_external._bless_my_loader(bar.__dict__))
+
+    def test_gh86298_loader_is_none_and_no_spec(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = None
+        del bar.__spec__
+        # 2022-10-06(warsaw): For backward compatibility with the
+        # implementation in _warnings.c, this can't raise an
+        # AttributeError.  See _bless_my_loader() in _bootstrap_external.py
+        # If working with a module:
+        ## self.assertRaises(
+        ##     AttributeError, _bootstrap_external._bless_my_loader,
+        ##     bar.__dict__)
+        self.assertIsNone(_bootstrap_external._bless_my_loader(bar.__dict__))
+
+    def test_gh86298_no_loader_and_spec_is_none(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        bar.__spec__ = None
+        self.assertRaises(
+            ValueError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_loader_is_none_and_spec_is_none(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = None
+        bar.__spec__ = None
+        self.assertRaises(
+            ValueError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_loader_is_none_and_spec_loader_is_none(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = None
+        bar.__spec__ = SimpleNamespace(loader=None)
+        self.assertRaises(
+            ValueError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_spec(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        del bar.__spec__
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_spec_is_none(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        bar.__spec__ = None
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_spec_loader(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        bar.__spec__ = SimpleNamespace()
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_loader_and_spec_loader_disagree(self):
+        bar = ModuleType('bar')
+        bar.__loader__ = object()
+        bar.__spec__ = SimpleNamespace(loader=object())
+        with warnings.catch_warnings():
+            self.assertWarns(
+                DeprecationWarning,
+                _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_loader_and_no_spec_loader(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        bar.__spec__ = SimpleNamespace()
+        self.assertRaises(
+            AttributeError,
+            _bootstrap_external._bless_my_loader, bar.__dict__)
+
+    def test_gh86298_no_loader_with_spec_loader_okay(self):
+        bar = ModuleType('bar')
+        del bar.__loader__
+        loader = object()
+        bar.__spec__ = SimpleNamespace(loader=loader)
+        self.assertEqual(
+            _bootstrap_external._bless_my_loader(bar.__dict__),
+            loader)
+
+
 if __name__ == "__main__":
     unittest.main()

diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-04-14-04-40.gh-issue-86298.QVM7G1.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-04-14-04-40.gh-issue-86298.QVM7G1.rst
new file mode 100644 (file)
index 0000000..6e349d5
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-04-14-04-40.gh-issue-86298.QVM7G1.rst
@@ -0,0 +1,3 @@
+In cases where ``warnings.warn_explicit()`` consults the module's loader, an
+``DeprecationWarning`` is issued when ``m.__loader__`` differs from
+``m.__spec__.loader``.

diff --git a/Python/_warnings.c b/Python/_warnings.c
index 1b9e107ea30b1378b5ce09d129ae482c3224ac40..0d4c50f769b03c67489d65bf8f56be54cd47e6c7 100644 (file)
--- a/Python/_warnings.c
+++ b/Python/_warnings.c
@@ -977,6 +977,7 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category,
 static PyObject *
 get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno)
 {
+    PyObject *external;
     PyObject *loader;
     PyObject *module_name;
     PyObject *get_source;
@@ -984,12 +985,18 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno
     PyObject *source_list;
     PyObject *source_line;
 
-    /* Check/get the requisite pieces needed for the loader. */
-    loader = _PyDict_GetItemWithError(module_globals, &_Py_ID(__loader__));
+    /* stolen from import.c */
+    external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
+    if (external == NULL) {
+        return NULL;
+    }
+
+    loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL);
+    Py_DECREF(external);
     if (loader == NULL) {
         return NULL;
     }
-    Py_INCREF(loader);
+
     module_name = _PyDict_GetItemWithError(module_globals, &_Py_ID(__name__));
     if (!module_name) {
         Py_DECREF(loader);