GH-113655: Lower the C recursion limit on various platforms (GH-113944)

diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index ed7dd829d4b6f0dd6c918d323d1e304741ec0ef3..10913943c1140d9f5aaa80d43c3c0b02089e6f35 100644 (file)
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -224,10 +224,14 @@ struct _ts {
    // recursions, sometimes less. 500 is a more conservative limit.
 #  define Py_C_RECURSION_LIMIT 500
 #elif defined(__s390x__)
-#  define Py_C_RECURSION_LIMIT 1200
+#  define Py_C_RECURSION_LIMIT 800
+#elif defined(_WIN32)
+#  define Py_C_RECURSION_LIMIT 4000
+#elif defined(_Py_ADDRESS_SANITIZER)
+#  define Py_C_RECURSION_LIMIT 4000
 #else
    // This value is duplicated in Lib/test/support/__init__.py
-#  define Py_C_RECURSION_LIMIT 8000
+#  define Py_C_RECURSION_LIMIT 10000
 #endif
 
 

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index e5fb725a30b5b8456b2389fab14ff19261ac4d9c..8344dd1849c61d46faf7d364c43d516308b9b8b7 100644 (file)
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2377,7 +2377,10 @@ def _get_c_recursion_limit():
         return _testcapi.Py_C_RECURSION_LIMIT
     except (ImportError, AttributeError):
         # Originally taken from Include/cpython/pystate.h .
-        return 8000
+        if sys.platform == 'win32':
+            return 4000
+        else:
+            return 10000
 
 # The default C recursion limit.
 Py_C_RECURSION_LIMIT = _get_c_recursion_limit()

diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 64fcb02309de77eec3a958b3ae080d891d17d6b1..3789ac22e3899cf84204e748fb16f85e86551985 100644 (file)
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -1126,7 +1126,7 @@ class AST_Tests(unittest.TestCase):
     def test_ast_recursion_limit(self):
         fail_depth = support.EXCEEDS_RECURSION_LIMIT
         crash_depth = 100_000
-        success_depth = 1200
+        success_depth = int(support.Py_C_RECURSION_LIMIT * 0.8)
         if _testinternalcapi is not None:
             remaining = _testinternalcapi.get_c_recursion_remaining()
             success_depth = min(success_depth, remaining)

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 50629b22822245063f1716277a369b4d5dfb715c..9c36f053314f9f02099e11543357af9f85805f45 100644 (file)
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -623,12 +623,10 @@ class TestSpecifics(unittest.TestCase):
     @support.cpython_only
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     def test_compiler_recursion_limit(self):
-        # Expected limit is Py_C_RECURSION_LIMIT * 2
-        # Duplicating the limit here is a little ugly.
-        # Perhaps it should be exposed somewhere...
-        fail_depth = Py_C_RECURSION_LIMIT * 2 + 1
+        # Expected limit is Py_C_RECURSION_LIMIT
+        fail_depth = Py_C_RECURSION_LIMIT + 1
         crash_depth = Py_C_RECURSION_LIMIT * 100
-        success_depth = int(Py_C_RECURSION_LIMIT * 1.8)
+        success_depth = int(Py_C_RECURSION_LIMIT * 0.8)
 
         def check_limit(prefix, repeated, mode="single"):
             expect_ok = prefix + repeated * success_depth

diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
index 0ef45d3c670e851213b655a7c221165bd06f9b50..7c66b906d308baf70e5e7a82b2200391bacca1ba 100644 (file)
--- a/Lib/test/test_functools.py
+++ b/Lib/test/test_functools.py
@@ -1875,8 +1875,14 @@ class TestLRU:
             return fib(n-1) + fib(n-2)
 
         if not support.Py_DEBUG:
+            depth = support.Py_C_RECURSION_LIMIT*2//7
             with support.infinite_recursion():
-                fib(2500)
+                fib(depth)
+        if self.module == c_functools:
+            fib.cache_clear()
+            with support.infinite_recursion():
+                with self.assertRaises(RecursionError):
+                    fib(10000)
 
 
 @py_functools.lru_cache()

diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index fc5ca72236b1fb700cd4dede8052d404d4954586..ae6e192a7ab6effa6252f7f57ebe916ae7cb8f07 100644 (file)
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -3037,10 +3037,8 @@ class TestExtendedArgs(unittest.TestCase):
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
     def test_trace_lots_of_globals(self):
-        count = 1000
-        if _testinternalcapi is not None:
-            remaining = _testinternalcapi.get_c_recursion_remaining()
-            count = min(count, remaining)
+
+        count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8))
 
         code = """if 1:
             def f():

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst
new file mode 100644 (file)
index 0000000..30f1b0a
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst
@@ -0,0 +1,3 @@
+Set the C recursion limit to 4000 on Windows, and 10000 on Linux/OSX. This
+seems to be near the sweet spot to maintain safety, but not compromise
+backwards compatibility.

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 4bb337349748cf7b3c9318edb599726b234526ee..ce92672bf00776aabe26d87a21200ff48b1fe523 100755 (executable)
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1388,15 +1388,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
 
     int starting_recursion_depth;
     /* Be careful here to prevent overflow. */
-    int COMPILER_STACK_FRAME_SCALE = 2;
     PyThreadState *tstate = _PyThreadState_GET();
     if (!tstate) {
         return NULL;
     }
     struct validator vstate;
-    vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    vstate.recursion_limit = Py_C_RECURSION_LIMIT;
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     vstate.recursion_depth = starting_recursion_depth;
 
     PyObject *result = ast2obj_mod(state, &vstate, t);

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 699e1c157c591caaa74362d49f92c905c863061c..d77e986ba067a3eccd636a4fe370d631717ffbb1 100644 (file)
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -13149,15 +13149,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
 
     int starting_recursion_depth;
     /* Be careful here to prevent overflow. */
-    int COMPILER_STACK_FRAME_SCALE = 2;
     PyThreadState *tstate = _PyThreadState_GET();
     if (!tstate) {
         return NULL;
     }
     struct validator vstate;
-    vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    vstate.recursion_limit = Py_C_RECURSION_LIMIT;
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     vstate.recursion_depth = starting_recursion_depth;
 
     PyObject *result = ast2obj_mod(state, &vstate, t);

diff --git a/Python/ast.c b/Python/ast.c
index 5f46d4149c2ed0b990ba2389a5d4bbfa487947a3..71b09d889f17c195eea668f6cfbf1116decffe88 100644 (file)
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -1037,10 +1037,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
     return 1;
 }
 
-
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
 int
 _PyAST_Validate(mod_ty mod)
 {
@@ -1057,9 +1053,9 @@ _PyAST_Validate(mod_ty mod)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     state.recursion_depth = starting_recursion_depth;
-    state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    state.recursion_limit = Py_C_RECURSION_LIMIT;
 
     switch (mod->kind) {
     case Module_kind:

diff --git a/Python/ast_opt.c b/Python/ast_opt.c
index 04d7ae6eaafbc0eeda98e7355fc6be7cd84dc629..41e906c66e8eec0e4611cf94d1eb586d5fa9ebf5 100644 (file)
--- a/Python/ast_opt.c
+++ b/Python/ast_opt.c
@@ -1100,9 +1100,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
 #undef CALL_OPT
 #undef CALL_SEQ
 
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
 int
 _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features)
 {
@@ -1120,9 +1117,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     state.recursion_depth = starting_recursion_depth;
-    state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    state.recursion_limit = Py_C_RECURSION_LIMIT;
 
     int ret = astfold_mod(mod, arena, &state);
     assert(ret || PyErr_Occurred());

diff --git a/Python/symtable.c b/Python/symtable.c
index 83137b491f282cb9e074b1504b74644a74ac3d34..743029956e32fafcb5530696732d7bdc12797e9c 100644 (file)
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -386,11 +386,6 @@ symtable_new(void)
     return NULL;
 }
 
-/* Using a scaling factor means this should automatically adjust when
-   the recursion limit is adjusted for small or large C stack allocations.
-*/
-#define COMPILER_STACK_FRAME_SCALE 2
-
 struct symtable *
 _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
 {
@@ -417,9 +412,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     st->recursion_depth = starting_recursion_depth;
-    st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    st->recursion_limit = Py_C_RECURSION_LIMIT;
 
     /* Make the initial symbol information gathering pass */
     if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {

diff --git a/Python/traceback.c b/Python/traceback.c
index abd429ac6c1f71f07e37e1f2e98f75a774257b49..7a188e56c939c0184114d61c912564c725f64c74 100644 (file)
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -965,7 +965,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
     unsigned int depth = 0;
     while (1) {
         if (MAX_FRAME_DEPTH <= depth) {
-            PUTS(fd, "  ...\n");
+            if (MAX_FRAME_DEPTH < depth) {
+                PUTS(fd, "plus ");
+                _Py_DumpDecimal(fd, depth);
+                PUTS(fd, " frames\n");
+            }
             break;
         }
         dump_frame(fd, frame);