bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856)

Importing ProcessPoolExecutor may hang or cause an error when the import
accesses urandom on a low resource platform

https://bugs.python.org/issue29877
(cherry picked from commit 1d817e4c8259f49602eefe9729743f6d9d748e8d)

Co-authored-by: Dustin Spicuzza <dustin@virtualroadside.com>

diff --git a/Lib/compileall.py b/Lib/compileall.py
index 1c9ceb693096a73e45cbbf9f6b0c79d74972b549..39210464b3422d3f8749d3576c0405bd82669bba 100644 (file)
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -16,10 +16,6 @@ import importlib.util
 import py_compile
 import struct
 
-try:
-    from concurrent.futures import ProcessPoolExecutor
-except ImportError:
-    ProcessPoolExecutor = None
 from functools import partial
 
 __all__ = ["compile_dir","compile_file","compile_path"]
@@ -68,9 +64,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
     optimize:  optimization level or -1 for level of the interpreter
     workers:   maximum number of parallel workers
     """
-    if workers is not None and workers < 0:
-        raise ValueError('workers must be greater or equal to 0')
-
+    ProcessPoolExecutor = None
+    if workers is not None:
+        if workers < 0:
+            raise ValueError('workers must be greater or equal to 0')
+        elif workers != 1:
+            try:
+                # Only import when needed, as low resource platforms may
+                # fail to import it
+                from concurrent.futures import ProcessPoolExecutor
+            except ImportError:
+                workers = 1
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
                       ddir=ddir)
     success = True

diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 2356efcaec78bebb5d1c06284e1e9aca63af8ee9..66a2004833fbf49be48c581c3b15897089159ddc 100644 (file)
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -161,7 +161,7 @@ class CompileallTests(unittest.TestCase):
         self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)')
         self.assertTrue(os.path.isfile(self.bc_path))
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     def test_compile_pool_called(self, pool_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)
         self.assertTrue(pool_mock.called)
@@ -171,19 +171,19 @@ class CompileallTests(unittest.TestCase):
                                     "workers must be greater or equal to 0"):
             compileall.compile_dir(self.directory, workers=-1)
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     def test_compile_workers_cpu_count(self, pool_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=0)
         self.assertEqual(pool_mock.call_args[1]['max_workers'], None)
 
-    @mock.patch('compileall.ProcessPoolExecutor')
+    @mock.patch('concurrent.futures.ProcessPoolExecutor')
     @mock.patch('compileall.compile_file')
     def test_compile_one_worker(self, compile_file_mock, pool_mock):
         compileall.compile_dir(self.directory, quiet=True)
         self.assertFalse(pool_mock.called)
         self.assertTrue(compile_file_mock.called)
 
-    @mock.patch('compileall.ProcessPoolExecutor', new=None)
+    @mock.patch('concurrent.futures.ProcessPoolExecutor', new=None)
     @mock.patch('compileall.compile_file')
     def test_compile_missing_multiprocessing(self, compile_file_mock):
         compileall.compile_dir(self.directory, quiet=True, workers=5)

diff --git a/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
new file mode 100644 (file)
index 0000000..cc09533
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
@@ -0,0 +1,2 @@
+compileall: import ProcessPoolExecutor only when needed, preventing hangs on
+low resource platforms