bpo-45020: Add -X frozen_modules=[on|off] to explicitly control use of frozen modules...

Currently we freeze several modules into the runtime. For each of these modules it is essential to bootstrapping the runtime that they be frozen. Any other stdlib module that we later freeze into the runtime is not essential. We can just as well import from the .py file.  This PR lets users explicitly choose which should be used, with the new "-X frozen_modules=[on|off]" CLI flag. The default is "off" for now.

https://bugs.python.org/issue45020

diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index 7ebc34f47fc39b3592c5d82cb6d9d92069ac3f3a..2398ab0d0cf262a38a663e5982dfef756db4c8c7 100644 (file)
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -480,6 +480,12 @@ Miscellaneous options
      objects and pyc files are desired as well as supressing the extra visual
      location indicators when the interpreter displays tracebacks. See also
      :envvar:`PYTHONNODEBUGRANGES`.
+   * ``-X frozen_modules`` determines whether or not frozen modules are
+     ignored by the import machinery.  A value of "on" means they get
+     imported and "off" means they are ignored.  The default is "on"
+     for non-debug builds (the normal case) and "off" for debug builds.
+     Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
+     frozen modules are always used, even if this flag is set to "off".
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -518,6 +524,9 @@ Miscellaneous options
    .. versionadded:: 3.11
       The ``-X no_debug_ranges`` option.
 
+   .. versionadded:: 3.11
+      The ``-X frozen_modules`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index 22ad0f14e58004cdc560456883e68abc68670880..65d52c45783f1856328730ce2d995d308df0a627 100644 (file)
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -172,6 +172,7 @@ typedef struct PyConfig {
     int legacy_windows_stdio;
 #endif
     wchar_t *check_hash_pycs_mode;
+    int use_frozen_modules;
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index 4b009e816b49279a9406e46472c4c330315d01d4..9014fcd41d868615368799acf0df986689dde5ba 100644 (file)
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy(
 extern PyStatus _PyConfig_InitPathConfig(
     PyConfig *config,
     int compute_path_config);
+extern PyStatus _PyConfig_InitImportConfig(PyConfig *config);
 extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);

diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index bfd082b588256bbe91bc910d072c8f034bb9dd92..0e6edf4ec2670239decb5e78c471b10cc1322a1d 100644 (file)
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -246,6 +246,9 @@ struct _is {
     PyObject *builtins;
     // importlib module
     PyObject *importlib;
+    // override for config->use_frozen_modules (for tests)
+    // (-1: "off", 1: "on", 0: no override)
+    int override_frozen_modules;
 
     /* Used in Modules/_threadmodule.c. */
     long num_threads;

diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py
index aa31d44184145baafb1d2da009665c05a16ff1a3..03bc9bba317571cf1124afe3e7a1643c8530b9ca 100644 (file)
--- a/Lib/ctypes/test/test_values.py
+++ b/Lib/ctypes/test/test_values.py
@@ -72,14 +72,16 @@ class PythonValuesTestCase(unittest.TestCase):
                 self.assertGreater(abs(entry.size), 10)
                 self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
                 # Check the module's package-ness.
-                spec = importlib.util.find_spec(modname)
+                with import_helper.frozen_modules():
+                    spec = importlib.util.find_spec(modname)
                 if entry.size < 0:
                     # It's a package.
                     self.assertIsNotNone(spec.submodule_search_locations)
                 else:
                     self.assertIsNone(spec.submodule_search_locations)
 
-        expected = imp._frozen_module_names()
+        with import_helper.frozen_modules():
+            expected = imp._frozen_module_names()
         self.maxDiff = None
         self.assertEqual(modules, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")

diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py
index 5d1e9406879cc613ac3aff646baf69aad204a559..10f745aa6b1448f3b23d70f1939c10b9c64d0e63 100644 (file)
--- a/Lib/test/support/import_helper.py
+++ b/Lib/test/support/import_helper.py
@@ -1,4 +1,5 @@
 import contextlib
+import _imp
 import importlib
 import importlib.util
 import os
@@ -109,7 +110,24 @@ def _save_and_block_module(name, orig_modules):
     return saved
 
 
-def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+@contextlib.contextmanager
+def frozen_modules(enabled=True):
+    """Force frozen modules to be used (or not).
+
+    This only applies to modules that haven't been imported yet.
+    Also, some essential modules will always be imported frozen.
+    """
+    _imp._override_frozen_modules_for_tests(1 if enabled else -1)
+    try:
+        yield
+    finally:
+        _imp._override_frozen_modules_for_tests(0)
+
+
+def import_fresh_module(name, fresh=(), blocked=(), *,
+                        deprecated=False,
+                        usefrozen=False,
+                        ):
     """Import and return a module, deliberately bypassing sys.modules.
 
     This function imports and returns a fresh copy of the named Python module
@@ -133,6 +151,9 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
 
     This function will raise ImportError if the named module cannot be
     imported.
+
+    If "usefrozen" is False (the default) then the frozen importer is
+    disabled (except for essential modules like importlib._bootstrap).
     """
     # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
     # to make sure that this utility function is working as expected
@@ -148,7 +169,8 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
             for blocked_name in blocked:
                 if not _save_and_block_module(blocked_name, orig_modules):
                     names_to_remove.append(blocked_name)
-            fresh_module = importlib.import_module(name)
+            with frozen_modules(usefrozen):
+                fresh_module = importlib.import_module(name)
         except ImportError:
             fresh_module = None
         finally:
@@ -169,9 +191,12 @@ class CleanImport(object):
 
         with CleanImport("foo"):
             importlib.import_module("foo") # new reference
+
+    If "usefrozen" is False (the default) then the frozen importer is
+    disabled (except for essential modules like importlib._bootstrap).
     """
 
-    def __init__(self, *module_names):
+    def __init__(self, *module_names, usefrozen=False):
         self.original_modules = sys.modules.copy()
         for module_name in module_names:
             if module_name in sys.modules:
@@ -183,12 +208,15 @@ class CleanImport(object):
                 if module.__name__ != module_name:
                     del sys.modules[module.__name__]
                 del sys.modules[module_name]
+        self._frozen_modules = frozen_modules(usefrozen)
 
     def __enter__(self):
+        self._frozen_modules.__enter__()
         return self
 
     def __exit__(self, *ignore_exc):
         sys.modules.update(self.original_modules)
+        self._frozen_modules.__exit__(*ignore_exc)
 
 
 class DirsOnSysPath(object):

diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py
index d9807a1e114b654285688f366b172e0a35599c8e..ce01417ed07d8849536983b411357615c0dc22b7 100644 (file)
--- a/Lib/test/support/os_helper.py
+++ b/Lib/test/support/os_helper.py
@@ -599,6 +599,10 @@ class EnvironmentVarGuard(collections.abc.MutableMapping):
     def unset(self, envvar):
         del self[envvar]
 
+    def copy(self):
+        # We do what os.environ.copy() does.
+        return dict(self)
+
     def __enter__(self):
         return self
 

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 8e3dd50c1f8be0af6ce8cb0803290e6895d9b6b6..e5e7c833e10d34d403b54c7c840b7796d7b6d68c 100644 (file)
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -12,6 +12,7 @@ import re
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -426,6 +427,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pathconfig_warnings': 1,
         '_init_main': 1,
         '_isolated_interpreter': 0,
+        'use_frozen_modules': False,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({

diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py
index 142f17d518e783e74742ea63bc62828c4cb61fa0..52d8f7ced96803375fc87f3884e9cfa99aa6f632 100644 (file)
--- a/Lib/test/test_frozen.py
+++ b/Lib/test/test_frozen.py
@@ -12,7 +12,7 @@
 
 import sys
 import unittest
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
 
 
 class TestFrozen(unittest.TestCase):
@@ -20,8 +20,9 @@ class TestFrozen(unittest.TestCase):
         name = '__hello__'
         if name in sys.modules:
             del sys.modules[name]
-        with captured_stdout() as out:
-            import __hello__
+        with import_helper.frozen_modules():
+            with captured_stdout() as out:
+                import __hello__
         self.assertEqual(out.getvalue(), 'Hello world!\n')
 
 

diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py
index eb7a4d275cfd263a74b477cef33b5aa66c2ff457..fbc3fc0d547ff7ae7c4866c1aa918dd3b67fee7e 100644 (file)
--- a/Lib/test/test_importlib/frozen/test_finder.py
+++ b/Lib/test/test_importlib/frozen/test_finder.py
@@ -6,6 +6,8 @@ machinery = util.import_importlib('importlib.machinery')
 import unittest
 import warnings
 
+from test.support import import_helper
+
 
 class FindSpecTests(abc.FinderTests):
 
@@ -13,7 +15,8 @@ class FindSpecTests(abc.FinderTests):
 
     def find(self, name, path=None):
         finder = self.machinery.FrozenImporter
-        return finder.find_spec(name, path)
+        with import_helper.frozen_modules():
+            return finder.find_spec(name, path)
 
     def test_module(self):
         name = '__hello__'
@@ -52,7 +55,8 @@ class FinderTests(abc.FinderTests):
         finder = self.machinery.FrozenImporter
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)
-            return finder.find_module(name, path)
+            with import_helper.frozen_modules():
+                return finder.find_module(name, path)
 
     def test_module(self):
         name = '__hello__'

diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py
index f0cf17990b372dd007efba4873cb23d257e82ae0..1b0a56f7e8afa980753c1a58c2e1465d7ec88e1e 100644 (file)
--- a/Lib/test/test_importlib/frozen/test_loader.py
+++ b/Lib/test/test_importlib/frozen/test_loader.py
@@ -3,27 +3,54 @@ from .. import util
 
 machinery = util.import_importlib('importlib.machinery')
 
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
+import contextlib
 import types
 import unittest
 import warnings
 
 
+@contextlib.contextmanager
+def deprecated():
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', DeprecationWarning)
+        yield
+
+
+@contextlib.contextmanager
+def fresh(name, *, oldapi=False):
+    with util.uncache(name):
+        with import_helper.frozen_modules():
+            with captured_stdout() as stdout:
+                if oldapi:
+                    with deprecated():
+                        yield stdout
+                else:
+                    yield stdout
+
+
 class ExecModuleTests(abc.LoaderTests):
 
     def exec_module(self, name):
-        with util.uncache(name), captured_stdout() as stdout:
-            spec = self.machinery.ModuleSpec(
-                    name, self.machinery.FrozenImporter, origin='frozen',
-                    is_package=self.machinery.FrozenImporter.is_package(name))
-            module = types.ModuleType(name)
-            module.__spec__ = spec
-            assert not hasattr(module, 'initialized')
+        with import_helper.frozen_modules():
+            is_package = self.machinery.FrozenImporter.is_package(name)
+        spec = self.machinery.ModuleSpec(
+            name,
+            self.machinery.FrozenImporter,
+            origin='frozen',
+            is_package=is_package,
+        )
+        module = types.ModuleType(name)
+        module.__spec__ = spec
+        assert not hasattr(module, 'initialized')
+
+        with fresh(name) as stdout:
             self.machinery.FrozenImporter.exec_module(module)
-            self.assertTrue(module.initialized)
-            self.assertTrue(hasattr(module, '__spec__'))
-            self.assertEqual(module.__spec__.origin, 'frozen')
-            return module, stdout.getvalue()
+
+        self.assertTrue(module.initialized)
+        self.assertTrue(hasattr(module, '__spec__'))
+        self.assertEqual(module.__spec__.origin, 'frozen')
+        return module, stdout.getvalue()
 
     def test_module(self):
         name = '__hello__'
@@ -50,20 +77,19 @@ class ExecModuleTests(abc.LoaderTests):
         name = '__phello__.spam'
         with util.uncache('__phello__'):
             module, output = self.exec_module(name)
-            check = {'__name__': name}
-            for attr, value in check.items():
-                attr_value = getattr(module, attr)
-                self.assertEqual(attr_value, value,
-                        'for {name}.{attr}, {given} != {expected!r}'.format(
-                                 name=name, attr=attr, given=attr_value,
-                                 expected=value))
-            self.assertEqual(output, 'Hello world!\n')
+        check = {'__name__': name}
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                    'for {name}.{attr}, {given} != {expected!r}'.format(
+                             name=name, attr=attr, given=attr_value,
+                             expected=value))
+        self.assertEqual(output, 'Hello world!\n')
 
     def test_module_repr(self):
         name = '__hello__'
         module, output = self.exec_module(name)
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', DeprecationWarning)
+        with deprecated():
             repr_str = self.machinery.FrozenImporter.module_repr(module)
         self.assertEqual(repr_str,
                          "<module '__hello__' (frozen)>")
@@ -78,7 +104,8 @@ class ExecModuleTests(abc.LoaderTests):
     test_state_after_failure = None
 
     def test_unloadable(self):
-        assert self.machinery.FrozenImporter.find_spec('_not_real') is None
+        with import_helper.frozen_modules():
+            assert self.machinery.FrozenImporter.find_spec('_not_real') is None
         with self.assertRaises(ImportError) as cm:
             self.exec_module('_not_real')
         self.assertEqual(cm.exception.name, '_not_real')
@@ -91,84 +118,76 @@ class ExecModuleTests(abc.LoaderTests):
 
 class LoaderTests(abc.LoaderTests):
 
+    def load_module(self, name):
+        with fresh(name, oldapi=True) as stdout:
+            module = self.machinery.FrozenImporter.load_module(name)
+        return module, stdout
+
     def test_module(self):
-        with util.uncache('__hello__'), captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__hello__')
-            check = {'__name__': '__hello__',
-                    '__package__': '',
-                    '__loader__': self.machinery.FrozenImporter,
-                    }
-            for attr, value in check.items():
-                self.assertEqual(getattr(module, attr), value)
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-            self.assertFalse(hasattr(module, '__file__'))
+        module, stdout = self.load_module('__hello__')
+        check = {'__name__': '__hello__',
+                '__package__': '',
+                '__loader__': self.machinery.FrozenImporter,
+                }
+        for attr, value in check.items():
+            self.assertEqual(getattr(module, attr), value)
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+        self.assertFalse(hasattr(module, '__file__'))
 
     def test_package(self):
-        with util.uncache('__phello__'),  captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__phello__')
-            check = {'__name__': '__phello__',
-                     '__package__': '__phello__',
-                     '__path__': [],
-                     '__loader__': self.machinery.FrozenImporter,
-                     }
-            for attr, value in check.items():
-                attr_value = getattr(module, attr)
-                self.assertEqual(attr_value, value,
-                                 "for __phello__.%s, %r != %r" %
-                                 (attr, attr_value, value))
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-            self.assertFalse(hasattr(module, '__file__'))
+        module, stdout = self.load_module('__phello__')
+        check = {'__name__': '__phello__',
+                 '__package__': '__phello__',
+                 '__path__': [],
+                 '__loader__': self.machinery.FrozenImporter,
+                 }
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                             "for __phello__.%s, %r != %r" %
+                             (attr, attr_value, value))
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+        self.assertFalse(hasattr(module, '__file__'))
 
     def test_lacking_parent(self):
-        with util.uncache('__phello__', '__phello__.spam'), \
-             captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__phello__.spam')
-            check = {'__name__': '__phello__.spam',
-                    '__package__': '__phello__',
-                    '__loader__': self.machinery.FrozenImporter,
-                    }
-            for attr, value in check.items():
-                attr_value = getattr(module, attr)
-                self.assertEqual(attr_value, value,
-                                 "for __phello__.spam.%s, %r != %r" %
-                                 (attr, attr_value, value))
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-            self.assertFalse(hasattr(module, '__file__'))
+        with util.uncache('__phello__'):
+            module, stdout = self.load_module('__phello__.spam')
+        check = {'__name__': '__phello__.spam',
+                '__package__': '__phello__',
+                '__loader__': self.machinery.FrozenImporter,
+                }
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                             "for __phello__.spam.%s, %r != %r" %
+                             (attr, attr_value, value))
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+        self.assertFalse(hasattr(module, '__file__'))
 
     def test_module_reuse(self):
-        with util.uncache('__hello__'), captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module1 = self.machinery.FrozenImporter.load_module('__hello__')
-                module2 = self.machinery.FrozenImporter.load_module('__hello__')
-            self.assertIs(module1, module2)
-            self.assertEqual(stdout.getvalue(),
-                             'Hello world!\nHello world!\n')
+        with fresh('__hello__', oldapi=True) as stdout:
+            module1 = self.machinery.FrozenImporter.load_module('__hello__')
+            module2 = self.machinery.FrozenImporter.load_module('__hello__')
+        self.assertIs(module1, module2)
+        self.assertEqual(stdout.getvalue(),
+                         'Hello world!\nHello world!\n')
 
     def test_module_repr(self):
-        with util.uncache('__hello__'), captured_stdout():
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__hello__')
-                repr_str = self.machinery.FrozenImporter.module_repr(module)
-            self.assertEqual(repr_str,
-                             "<module '__hello__' (frozen)>")
+        with fresh('__hello__', oldapi=True) as stdout:
+            module = self.machinery.FrozenImporter.load_module('__hello__')
+            repr_str = self.machinery.FrozenImporter.module_repr(module)
+        self.assertEqual(repr_str,
+                         "<module '__hello__' (frozen)>")
 
     # No way to trigger an error in a frozen module.
     test_state_after_failure = None
 
     def test_unloadable(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore", DeprecationWarning)
-            assert self.machinery.FrozenImporter.find_module('_not_real') is None
+        with import_helper.frozen_modules():
+            with deprecated():
+                assert self.machinery.FrozenImporter.find_module('_not_real') is None
             with self.assertRaises(ImportError) as cm:
-                self.machinery.FrozenImporter.load_module('_not_real')
+                self.load_module('_not_real')
             self.assertEqual(cm.exception.name, '_not_real')
 
 
@@ -185,15 +204,17 @@ class InspectLoaderTests:
         # Make sure that the code object is good.
         name = '__hello__'
         with captured_stdout() as stdout:
-            code = self.machinery.FrozenImporter.get_code(name)
-            mod = types.ModuleType(name)
-            exec(code, mod.__dict__)
-            self.assertTrue(hasattr(mod, 'initialized'))
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            with import_helper.frozen_modules():
+                code = self.machinery.FrozenImporter.get_code(name)
+                mod = types.ModuleType(name)
+                exec(code, mod.__dict__)
+        self.assertTrue(hasattr(mod, 'initialized'))
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
     def test_get_source(self):
         # Should always return None.
-        result = self.machinery.FrozenImporter.get_source('__hello__')
+        with import_helper.frozen_modules():
+            result = self.machinery.FrozenImporter.get_source('__hello__')
         self.assertIsNone(result)
 
     def test_is_package(self):
@@ -201,7 +222,8 @@ class InspectLoaderTests:
         test_for = (('__hello__', False), ('__phello__', True),
                     ('__phello__.spam', False))
         for name, is_package in test_for:
-            result = self.machinery.FrozenImporter.is_package(name)
+            with import_helper.frozen_modules():
+                result = self.machinery.FrozenImporter.is_package(name)
             self.assertEqual(bool(result), is_package)
 
     def test_failure(self):
@@ -209,7 +231,8 @@ class InspectLoaderTests:
         for meth_name in ('get_code', 'get_source', 'is_package'):
             method = getattr(self.machinery.FrozenImporter, meth_name)
             with self.assertRaises(ImportError) as cm:
-                method('importlib')
+                with import_helper.frozen_modules():
+                    method('importlib')
             self.assertEqual(cm.exception.name, 'importlib')
 
 (Frozen_ILTests,

diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst
new file mode 100644 (file)
index 0000000..f6dffa0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst
@@ -0,0 +1,3 @@
+Add a new command line option, "-X frozen_modules=[on|off]" to opt out
+of (or into) using optional frozen modules.  This defaults to "on" (or
+"off" if it's a debug build).

diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h
index ec4ebca36d943006a911f8e726700d6809525f8e..438a348fa097fb3e3e999f4af52bfe90f5473742 100644 (file)
--- a/Python/clinic/import.c.h
+++ b/Python/clinic/import.c.h
@@ -315,6 +315,37 @@ _imp__frozen_module_names(PyObject *module, PyObject *Py_UNUSED(ignored))
     return _imp__frozen_module_names_impl(module);
 }
 
+PyDoc_STRVAR(_imp__override_frozen_modules_for_tests__doc__,
+"_override_frozen_modules_for_tests($module, override, /)\n"
+"--\n"
+"\n"
+"(internal-only) Override PyConfig.use_frozen_modules.\n"
+"\n"
+"(-1: \"off\", 1: \"on\", 0: no override)\n"
+"See frozen_modules() in Lib/test/support/import_helper.py.");
+
+#define _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF    \
+    {"_override_frozen_modules_for_tests", (PyCFunction)_imp__override_frozen_modules_for_tests, METH_O, _imp__override_frozen_modules_for_tests__doc__},
+
+static PyObject *
+_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override);
+
+static PyObject *
+_imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg)
+{
+    PyObject *return_value = NULL;
+    int override;
+
+    override = _PyLong_AsInt(arg);
+    if (override == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = _imp__override_frozen_modules_for_tests_impl(module, override);
+
+exit:
+    return return_value;
+}
+
 #if defined(HAVE_DYNAMIC_LOADING)
 
 PyDoc_STRVAR(_imp_create_dynamic__doc__,
@@ -467,4 +498,4 @@ exit:
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=0ab3fa7c5808bba4 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=96038c277119d6e3 input=a9049054013a1b77]*/

diff --git a/Python/import.c b/Python/import.c
index d896ff476e17969de343ef1762cc2f6d1954702a..317a836617c51d0434ba1d9f1c416217488bce25 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -1050,17 +1050,58 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
 
 /* Frozen modules */
 
+static bool
+is_essential_frozen_module(const char *name)
+{
+    /* These modules are necessary to bootstrap the import system. */
+    if (strcmp(name, "_frozen_importlib") == 0) {
+        return true;
+    }
+    if (strcmp(name, "_frozen_importlib_external") == 0) {
+        return true;
+    }
+    if (strcmp(name, "zipimport") == 0) {
+        return true;
+    }
+    /* This doesn't otherwise have anywhere to find the module.
+       See frozenmain.c. */
+    if (strcmp(name, "__main__") == 0) {
+        return true;
+    }
+    return false;
+}
+
+static bool
+use_frozen(void)
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    int override = interp->override_frozen_modules;
+    if (override > 0) {
+        return true;
+    }
+    else if (override < 0) {
+        return false;
+    }
+    else {
+        return interp->config.use_frozen_modules;
+    }
+}
+
 static PyObject *
-list_frozen_module_names(bool force)
+list_frozen_module_names()
 {
     PyObject *names = PyList_New(0);
     if (names == NULL) {
         return NULL;
     }
+    bool enabled = use_frozen();
     for (const struct _frozen *p = PyImport_FrozenModules; ; p++) {
         if (p->name == NULL) {
             break;
         }
+        if (!enabled && !is_essential_frozen_module(p->name)) {
+            continue;
+        }
         PyObject *name = PyUnicode_FromString(p->name);
         if (name == NULL) {
             Py_DECREF(names);
@@ -1077,18 +1118,27 @@ list_frozen_module_names(bool force)
 }
 
 static const struct _frozen *
-find_frozen(PyObject *name)
+find_frozen(PyObject *modname)
 {
-    const struct _frozen *p;
-
-    if (name == NULL)
+    if (modname == NULL) {
         return NULL;
-
+    }
+    const char *name = PyUnicode_AsUTF8(modname);
+    if (name == NULL) {
+        PyErr_Clear();
+        return NULL;
+    }
+    if (!use_frozen() && !is_essential_frozen_module(name)) {
+        return NULL;
+    }
+    const struct _frozen *p;
     for (p = PyImport_FrozenModules; ; p++) {
-        if (p->name == NULL)
+        if (p->name == NULL) {
             return NULL;
-        if (_PyUnicode_EqualToASCIIString(name, p->name))
+        }
+        if (strcmp(name, p->name) == 0) {
             break;
+        }
     }
     return p;
 }
@@ -1991,7 +2041,28 @@ static PyObject *
 _imp__frozen_module_names_impl(PyObject *module)
 /*[clinic end generated code: output=80609ef6256310a8 input=76237fbfa94460d2]*/
 {
-    return list_frozen_module_names(true);
+    return list_frozen_module_names();
+}
+
+/*[clinic input]
+_imp._override_frozen_modules_for_tests
+
+    override: int
+    /
+
+(internal-only) Override PyConfig.use_frozen_modules.
+
+(-1: "off", 1: "on", 0: no override)
+See frozen_modules() in Lib/test/support/import_helper.py.
+[clinic start generated code]*/
+
+static PyObject *
+_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override)
+/*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    interp->override_frozen_modules = override;
+    Py_RETURN_NONE;
 }
 
 /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */
@@ -2155,6 +2226,7 @@ static PyMethodDef imp_methods[] = {
     _IMP_IS_BUILTIN_METHODDEF
     _IMP_IS_FROZEN_METHODDEF
     _IMP__FROZEN_MODULE_NAMES_METHODDEF
+    _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF
     _IMP_CREATE_DYNAMIC_METHODDEF
     _IMP_EXEC_DYNAMIC_METHODDEF
     _IMP_EXEC_BUILTIN_METHODDEF

diff --git a/Python/initconfig.c b/Python/initconfig.c
index 61cd0e6213ed17bf81ce34c2d2b771c9d1d08230..8740cc1cf7a2b32961fad185ea6c613074a1cce2 100644 (file)
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -100,6 +100,8 @@ static const char usage_3[] = "\
             instruction in code objects. This is useful when smaller code objects and pyc \n\
             files are desired as well as supressing the extra visual location indicators \n\
             when the interpreter displays tracebacks.\n\
+         -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\
+            The default is \"on\" (or \"off\" if you are running a local build).\n\
 \n\
 --check-hash-based-pycs always|default|never:\n\
     control how Python invalidates hash-based .pyc files\n\
@@ -949,6 +951,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(pathconfig_warnings);
     COPY_ATTR(_init_main);
     COPY_ATTR(_isolated_interpreter);
+    COPY_ATTR(use_frozen_modules);
     COPY_WSTRLIST(orig_argv);
 
 #undef COPY_ATTR
@@ -1052,6 +1055,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(_init_main);
     SET_ITEM_INT(_isolated_interpreter);
     SET_ITEM_WSTRLIST(orig_argv);
+    SET_ITEM_INT(use_frozen_modules);
 
     return dict;
 
@@ -1334,6 +1338,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(_install_importlib);
     GET_UINT(_init_main);
     GET_UINT(_isolated_interpreter);
+    GET_UINT(use_frozen_modules);
 
 #undef CHECK_VALUE
 #undef GET_UINT
@@ -1590,6 +1595,17 @@ config_get_xoption(const PyConfig *config, wchar_t *name)
     return _Py_get_xoption(&config->xoptions, name);
 }
 
+static const wchar_t*
+config_get_xoption_value(const PyConfig *config, wchar_t *name)
+{
+    const wchar_t *xoption = config_get_xoption(config, name);
+    if (xoption == NULL) {
+        return NULL;
+    }
+    const wchar_t *sep = wcschr(xoption, L'=');
+    return sep ? sep + 1 : L"";
+}
+
 
 static PyStatus
 config_init_home(PyConfig *config)
@@ -2065,6 +2081,48 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
 }
 
 
+static PyStatus
+config_init_import(PyConfig *config, int compute_path_config)
+{
+    PyStatus status;
+
+    status = _PyConfig_InitPathConfig(config, compute_path_config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    /* -X frozen_modules=[on|off] */
+    const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
+    if (value == NULL) {
+        // For now we always default to "off".
+        // In the near future we will be factoring in PGO and in-development.
+        config->use_frozen_modules = 0;
+    }
+    else if (wcscmp(value, L"on") == 0) {
+        config->use_frozen_modules = 1;
+    }
+    else if (wcscmp(value, L"off") == 0) {
+        config->use_frozen_modules = 0;
+    }
+    else if (wcslen(value) == 0) {
+        // "-X frozen_modules" and "-X frozen_modules=" both imply "on".
+        config->use_frozen_modules = 1;
+    }
+    else {
+        return PyStatus_Error("bad value for option -X frozen_modules "
+                              "(expected \"on\" or \"off\")");
+    }
+
+    return _PyStatus_OK();
+}
+
+PyStatus
+_PyConfig_InitImportConfig(PyConfig *config)
+{
+    return config_init_import(config, 1);
+}
+
+
 static PyStatus
 config_read(PyConfig *config, int compute_path_config)
 {
@@ -2111,7 +2169,7 @@ config_read(PyConfig *config, int compute_path_config)
     }
 
     if (config->_install_importlib) {
-        status = _PyConfig_InitPathConfig(config, compute_path_config);
+        status = config_init_import(config, compute_path_config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index f3b6b0ac68a1ea98d8ac7589d72dc16cdffad859..b10a19c0d8d5f2af3389fc443195ef823507348d 100644 (file)
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1078,8 +1078,8 @@ init_interp_main(PyThreadState *tstate)
         return _PyStatus_OK();
     }
 
-    // Compute the path configuration
-    status = _PyConfig_InitPathConfig(&interp->config, 1);
+    // Initialize the import-related configuration.
+    status = _PyConfig_InitImportConfig(&interp->config);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }