]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
bpo-42500: Fix recursion in or after except (GH-23568) (#24501)
authorMark Shannon <mark@hotpy.org>
Tue, 2 Mar 2021 10:36:38 +0000 (10:36 +0000)
committerGitHub <noreply@github.com>
Tue, 2 Mar 2021 10:36:38 +0000 (11:36 +0100)
* Use counter, rather boolean state when handling soft overflows.

(cherry picked from commit 4e7a69bdb63a104587759d7784124492dcdd496e)

Include/cpython/pystate.h
Include/internal/pycore_ceval.h
Lib/test/test_exceptions.py
Lib/test/test_sys.py
Misc/NEWS.d/next/Core and Builtins/2020-11-30-14-27-29.bpo-42500.excVKU.rst [new file with mode: 0644]
Python/ceval.c
Python/errors.c
Python/pystate.c
Python/sysmodule.c

index f292da1d3c6c5ee48ceaad0639b9cb42d4414d9f..29f6bf5a662dcd51fd5dac9d77f9cb87f52e8f56 100644 (file)
@@ -58,8 +58,7 @@ struct _ts {
     /* Borrowed reference to the current frame (it can be NULL) */
     PyFrameObject *frame;
     int recursion_depth;
-    char overflowed; /* The stack has overflowed. Allow 50 more calls
-                        to handle the runtime error. */
+    int recursion_headroom; /* Allow 50 more calls to handle any errors. */
     char recursion_critical; /* The current calls must not cause
                                 a stack overflow. */
     int stackcheck_counter;
index 18c8f027af16e6410452648f7a110bd11cf86d78..e7ace9bd01c2a907d6865d6246e614e7044505f2 100644 (file)
@@ -90,24 +90,8 @@ static inline int _Py_EnterRecursiveCall_inline(const char *where) {
 
 #define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where)
 
-/* Compute the "lower-water mark" for a recursion limit. When
- * Py_LeaveRecursiveCall() is called with a recursion depth below this mark,
- * the overflowed flag is reset to 0. */
-static inline int _Py_RecursionLimitLowerWaterMark(int limit) {
-    if (limit > 200) {
-        return (limit - 50);
-    }
-    else {
-        return (3 * (limit >> 2));
-    }
-}
-
 static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate)  {
     tstate->recursion_depth--;
-    int limit = tstate->interp->ceval.recursion_limit;
-    if (tstate->recursion_depth < _Py_RecursionLimitLowerWaterMark(limit)) {
-        tstate->overflowed = 0;
-    }
 }
 
 static inline void _Py_LeaveRecursiveCall_inline(void)  {
index 8d125b57ad6d5a8b98f82e4f0b274f611b2bc9fc..9c09bdc317dacf9d8c9fe9e4677e55e3a1dca519 100644 (file)
@@ -1043,7 +1043,7 @@ class ExceptionTests(unittest.TestCase):
                 # tstate->recursion_depth is equal to (recursion_limit - 1)
                 # and is equal to recursion_limit when _gen_throw() calls
                 # PyErr_NormalizeException().
-                recurse(setrecursionlimit(depth + 2) - depth - 1)
+                recurse(setrecursionlimit(depth + 2) - depth)
             finally:
                 sys.setrecursionlimit(recursionlimit)
                 print('Done.')
@@ -1073,6 +1073,54 @@ class ExceptionTests(unittest.TestCase):
                       b'while normalizing an exception', err)
         self.assertIn(b'Done.', out)
 
+
+    def test_recursion_in_except_handler(self):
+
+        def set_relative_recursion_limit(n):
+            depth = 1
+            while True:
+                try:
+                    sys.setrecursionlimit(depth)
+                except RecursionError:
+                    depth += 1
+                else:
+                    break
+            sys.setrecursionlimit(depth+n)
+
+        def recurse_in_except():
+            try:
+                1/0
+            except:
+                recurse_in_except()
+
+        def recurse_after_except():
+            try:
+                1/0
+            except:
+                pass
+            recurse_after_except()
+
+        def recurse_in_body_and_except():
+            try:
+                recurse_in_body_and_except()
+            except:
+                recurse_in_body_and_except()
+
+        recursionlimit = sys.getrecursionlimit()
+        try:
+            set_relative_recursion_limit(10)
+            for func in (recurse_in_except, recurse_after_except, recurse_in_body_and_except):
+                with self.subTest(func=func):
+                    try:
+                        func()
+                    except RecursionError:
+                        pass
+                    else:
+                        self.fail("Should have raised a RecursionError")
+        finally:
+            sys.setrecursionlimit(recursionlimit)
+
+
     @cpython_only
     def test_recursion_normalizing_with_no_memory(self):
         # Issue #30697. Test that in the abort that occurs when there is no
@@ -1109,7 +1157,7 @@ class ExceptionTests(unittest.TestCase):
             except MemoryError as e:
                 tb = e.__traceback__
             else:
-                self.fail("Should have raises a MemoryError")
+                self.fail("Should have raised a MemoryError")
             return traceback.format_tb(tb)
 
         tb1 = raiseMemError()
index d79151355eab1d3a9d397bacfa8975fd03dd6b58..fc6c1ef1acd7cdf6e3878f0eb473c4397689501d 100644 (file)
@@ -219,7 +219,7 @@ class SysModuleTest(unittest.TestCase):
         def f():
             f()
         try:
-            for depth in (10, 25, 50, 75, 100, 250, 1000):
+            for depth in (50, 75, 100, 250, 1000):
                 try:
                     sys.setrecursionlimit(depth)
                 except RecursionError:
@@ -229,17 +229,17 @@ class SysModuleTest(unittest.TestCase):
 
                 # Issue #5392: test stack overflow after hitting recursion
                 # limit twice
-                self.assertRaises(RecursionError, f)
-                self.assertRaises(RecursionError, f)
+                with self.assertRaises(RecursionError):
+                    f()
+                with self.assertRaises(RecursionError):
+                    f()
         finally:
             sys.setrecursionlimit(oldlimit)
 
     @test.support.cpython_only
     def test_setrecursionlimit_recursion_depth(self):
         # Issue #25274: Setting a low recursion limit must be blocked if the
-        # current recursion depth is already higher than the "lower-water
-        # mark". Otherwise, it may not be possible anymore to
-        # reset the overflowed flag to 0.
+        # current recursion depth is already higher than limit.
 
         from _testinternalcapi import get_recursion_depth
 
@@ -260,42 +260,10 @@ class SysModuleTest(unittest.TestCase):
             sys.setrecursionlimit(1000)
 
             for limit in (10, 25, 50, 75, 100, 150, 200):
-                # formula extracted from _Py_RecursionLimitLowerWaterMark()
-                if limit > 200:
-                    depth = limit - 50
-                else:
-                    depth = limit * 3 // 4
-                set_recursion_limit_at_depth(depth, limit)
+                set_recursion_limit_at_depth(limit, limit)
         finally:
             sys.setrecursionlimit(oldlimit)
 
-    # The error message is specific to CPython
-    @test.support.cpython_only
-    def test_recursionlimit_fatalerror(self):
-        # A fatal error occurs if a second recursion limit is hit when recovering
-        # from a first one.
-        code = textwrap.dedent("""
-            import sys
-
-            def f():
-                try:
-                    f()
-                except RecursionError:
-                    f()
-
-            sys.setrecursionlimit(%d)
-            f()""")
-        with test.support.SuppressCrashReport():
-            for i in (50, 1000):
-                sub = subprocess.Popen([sys.executable, '-c', code % i],
-                    stderr=subprocess.PIPE)
-                err = sub.communicate()[1]
-                self.assertTrue(sub.returncode, sub.returncode)
-                self.assertIn(
-                    b"Fatal Python error: _Py_CheckRecursiveCall: "
-                    b"Cannot recover from stack overflow",
-                    err)
-
     def test_getwindowsversion(self):
         # Raise SkipTest if sys doesn't have getwindowsversion attribute
         test.support.get_attribute(sys, "getwindowsversion")
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-30-14-27-29.bpo-42500.excVKU.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-30-14-27-29.bpo-42500.excVKU.rst
new file mode 100644 (file)
index 0000000..2462a8e
--- /dev/null
@@ -0,0 +1,2 @@
+Improve handling of exceptions near recursion limit. Converts a number of
+Fatal Errors in RecursionErrors.
index 91e879e804204450251a59e4be0bccf012171222..b7176dc0f6b8a2db052517549a2f07f1d637dd26 100644 (file)
@@ -793,23 +793,22 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
         _Py_CheckRecursionLimit = recursion_limit;
     }
 #endif
-    if (tstate->recursion_critical)
-        /* Somebody asked that we don't check for recursion. */
-        return 0;
-    if (tstate->overflowed) {
+    if (tstate->recursion_headroom) {
         if (tstate->recursion_depth > recursion_limit + 50) {
             /* Overflowing while handling an overflow. Give up. */
             Py_FatalError("Cannot recover from stack overflow.");
         }
-        return 0;
     }
-    if (tstate->recursion_depth > recursion_limit) {
-        --tstate->recursion_depth;
-        tstate->overflowed = 1;
-        _PyErr_Format(tstate, PyExc_RecursionError,
-                      "maximum recursion depth exceeded%s",
-                      where);
-        return -1;
+    else {
+        if (tstate->recursion_depth > recursion_limit) {
+            tstate->recursion_headroom++;
+            _PyErr_Format(tstate, PyExc_RecursionError,
+                        "maximum recursion depth exceeded%s",
+                        where);
+            tstate->recursion_headroom--;
+            --tstate->recursion_depth;
+            return -1;
+        }
     }
     return 0;
 }
index 87af39d527a512668dfc16ab44baf0da01e3a6af..d8c2d8b9236ed63728966787960fc590e4381240 100644 (file)
@@ -290,12 +290,14 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
                           PyObject **val, PyObject **tb)
 {
     int recursion_depth = 0;
+    tstate->recursion_headroom++;
     PyObject *type, *value, *initial_tb;
 
   restart:
     type = *exc;
     if (type == NULL) {
         /* There was no exception, so nothing to do. */
+        tstate->recursion_headroom--;
         return;
     }
 
@@ -347,6 +349,7 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
     }
     *exc = type;
     *val = value;
+    tstate->recursion_headroom--;
     return;
 
   error:
index 9beefa8e20c444d1664f65c70782bf1239e61bdf..71aac6f695d6e8e32d7de5877bee0053b3aab167 100644 (file)
@@ -576,7 +576,7 @@ new_threadstate(PyInterpreterState *interp, int init)
 
     tstate->frame = NULL;
     tstate->recursion_depth = 0;
-    tstate->overflowed = 0;
+    tstate->recursion_headroom = 0;
     tstate->recursion_critical = 0;
     tstate->stackcheck_counter = 0;
     tstate->tracing = 0;
index 3e4115fe8e1f93676d067b0f91934b7992d008e5..4c7f7b67fa837b900f1a56a676de10f7a8d4c98b 100644 (file)
@@ -1160,7 +1160,6 @@ static PyObject *
 sys_setrecursionlimit_impl(PyObject *module, int new_limit)
 /*[clinic end generated code: output=35e1c64754800ace input=b0f7a23393924af3]*/
 {
-    int mark;
     PyThreadState *tstate = _PyThreadState_GET();
 
     if (new_limit < 1) {
@@ -1178,8 +1177,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit)
        Reject too low new limit if the current recursion depth is higher than
        the new low-water mark. Otherwise it may not be possible anymore to
        reset the overflowed flag to 0. */
-    mark = _Py_RecursionLimitLowerWaterMark(new_limit);
-    if (tstate->recursion_depth >= mark) {
+    if (tstate->recursion_depth >= new_limit) {
         _PyErr_Format(tstate, PyExc_RecursionError,
                       "cannot set the recursion limit to %i at "
                       "the recursion depth %i: the limit is too low",