From: Sam Gross Date: Tue, 19 Mar 2024 19:22:42 +0000 (-0400) Subject: [3.12] gh-113964: Don't prevent new threads until all non-daemon threads exit (GH... X-Git-Tag: v3.12.3~60 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=92564331defba3462116d54658cd97624bb12678;p=thirdparty%2FPython%2Fcpython.git [3.12] gh-113964: Don't prevent new threads until all non-daemon threads exit (GH-116677) (#117029) Starting in Python 3.12, we prevented calling fork() and starting new threads during interpreter finalization (shutdown). This has led to a number of regressions and flaky tests. We should not prevent starting new threads (or `fork()`) until all non-daemon threads exit and finalization starts in earnest. This changes the checks to use `_PyInterpreterState_GetFinalizing(interp)`, which is set immediately before terminating non-daemon threads. (cherry picked from commit 60e105c1c11ecca1680d03c38aa06bcc77a28714) --- diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 46aec219c136..e8f80046649c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4840,20 +4840,21 @@ class ForkTests(unittest.TestCase): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - def test_fork_at_exit(self): + def test_fork_at_finalization(self): code = """if 1: import atexit import os - def exit_handler(): - pid = os.fork() - if pid != 0: - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(b"", out) + self.assertEqual(b"OK\n", out) self.assertIn(b"can't fork at interpreter shutdown", err) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d83bd9be1406..79efa42791bb 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3413,14 +3413,15 @@ class POSIXProcessTestCase(BaseTestCase): def dummy(): pass - def exit_handler(): - subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 119b41f39245..2e4b860b975f 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1133,21 +1133,21 @@ class ThreadTests(BaseTestCase): self.assertEqual(out, b'') self.assertEqual(err, b'') - def test_start_new_thread_at_exit(self): + def test_start_new_thread_at_finalization(self): code = """if 1: - import atexit import _thread def f(): print("shouldn't be printed") - def exit_handler(): - _thread.start_new_thread(f, ()) - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) class ThreadJoinOnShutdown(BaseTestCase): @@ -1276,6 +1276,30 @@ class ThreadJoinOnShutdown(BaseTestCase): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst new file mode 100644 index 000000000000..ab370d4aa1ba --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst @@ -0,0 +1,2 @@ +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index d75bb92757c7..35ea2ac306a3 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -946,7 +946,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); PyInterpreterState *interp = PyInterpreterState_Get(); - if ((preexec_fn != Py_None) && interp->finalizing) { + if ((preexec_fn != Py_None) && + _PyInterpreterState_GetFinalizing(interp) != NULL) + { PyErr_SetString(PyExc_RuntimeError, "preexec_fn not supported at interpreter shutdown"); return NULL; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 568fe8375d1e..365f4460088a 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1190,7 +1190,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) "thread is not supported for isolated subinterpreters"); return NULL; } - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't create new thread at interpreter shutdown"); return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d665d069d2c4..9f69c792614e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7663,7 +7663,7 @@ os_fork1_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't fork at interpreter shutdown"); return NULL; @@ -7707,7 +7707,7 @@ os_fork_impl(PyObject *module) { pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't fork at interpreter shutdown"); return NULL; @@ -8391,7 +8391,7 @@ os_forkpty_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't fork at interpreter shutdown"); return NULL; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 65d0842cb111..fc9379be02f4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -516,7 +516,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) /* Disable checks during Python finalization. For example, it allows to call _PyObject_Dump() during finalization for debugging purpose. */ - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; }