[3.12] GH-112215: Backport C recursion changes (GH-115083)

diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 628f2e0996e4695b0d72516cf69d9fb1847eba32..95fad893786d8d9237e292d079f5aa632cb9a2f8 100644 (file)
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -251,12 +251,24 @@ struct _ts {
 /* WASI has limited call stack. Python's recursion limit depends on code
    layout, optimization, and WASI runtime. Wasmtime can handle about 700
    recursions, sometimes less. 500 is a more conservative limit. */
-#ifndef C_RECURSION_LIMIT
-#  ifdef __wasi__
+#ifdef Py_DEBUG
+#  if defined(__wasi__)
+#    define C_RECURSION_LIMIT 150
+#  else
+#    define C_RECURSION_LIMIT 500
+#  endif
+#else
+#  if defined(__wasi__)
 #    define C_RECURSION_LIMIT 500
+#  elif defined(__s390x__)
+#    define C_RECURSION_LIMIT 800
+#  elif defined(_WIN32)
+#    define C_RECURSION_LIMIT 3000
+#  elif defined(_Py_ADDRESS_SANITIZER)
+#    define C_RECURSION_LIMIT 4000
 #  else
-    // This value is duplicated in Lib/test/support/__init__.py
-#    define C_RECURSION_LIMIT 1500
+     // This value is duplicated in Lib/test/support/__init__.py
+#    define C_RECURSION_LIMIT 10000
 #  endif
 #endif
 

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 8c4b4e023f633fe2a9bc52aebf6abbe7e34ee79c..cb5a84aa74e05f331a1656765dcca59af41ff494 100644 (file)
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2112,13 +2112,13 @@ def set_recursion_limit(limit):
     finally:
         sys.setrecursionlimit(original_limit)
 
-def infinite_recursion(max_depth=100):
-    """Set a lower limit for tests that interact with infinite recursions
-    (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
-    debug windows builds, due to not enough functions being inlined the
-    stack size might not handle the default recursion limit (1000). See
-    bpo-11105 for details."""
-    if max_depth < 3:
+def infinite_recursion(max_depth=None):
+    if max_depth is None:
+        # Pick a number large enough to cause problems
+        # but not take too long for code that can handle
+        # very deep recursion.
+        max_depth = 20_000
+    elif max_depth < 3:
         raise ValueError("max_depth must be at least 3, got {max_depth}")
     depth = get_recursion_depth()
     depth = max(depth - 1, 1)  # Ignore infinite_recursion() frame.
@@ -2362,7 +2362,22 @@ def adjust_int_max_str_digits(max_digits):
 EXCEEDS_RECURSION_LIMIT = 5000
 
 # The default C recursion limit (from Include/cpython/pystate.h).
-C_RECURSION_LIMIT = 1500
+if Py_DEBUG:
+    if is_wasi:
+        C_RECURSION_LIMIT = 150
+    else:
+        C_RECURSION_LIMIT = 500
+else:
+    if is_wasi:
+        C_RECURSION_LIMIT = 500
+    elif hasattr(os, 'uname') and os.uname().machine == 's390x':
+        C_RECURSION_LIMIT = 800
+    elif sys.platform.startswith('win'):
+        C_RECURSION_LIMIT = 3000
+    elif check_sanitizer(address=True):
+        C_RECURSION_LIMIT = 4000
+    else:
+        C_RECURSION_LIMIT = 10000
 
 #Windows doesn't have os.uname() but it doesn't support s390x.
 skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',

diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 3ba7cf7b04266fc077a71ee7d7ca84b81602538a..9736208a92fc6e0b6610e1a1a68938affa6cceb9 100644 (file)
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -1087,9 +1087,9 @@ class AST_Tests(unittest.TestCase):
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     @support.cpython_only
     def test_ast_recursion_limit(self):
-        fail_depth = support.EXCEEDS_RECURSION_LIMIT
+        fail_depth = support.C_RECURSION_LIMIT + 1
         crash_depth = 100_000
-        success_depth = 1200
+        success_depth = int(support.C_RECURSION_LIMIT * 0.9)
 
         def check_limit(prefix, repeated):
             expect_ok = prefix + repeated * success_depth

diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py
index ec8dc29d36c16a659c89137ab90181497d700a30..46abf40605f4916a6e41b72e2af7f154a1b2f517 100644 (file)
--- a/Lib/test/test_call.py
+++ b/Lib/test/test_call.py
@@ -1,5 +1,5 @@
 import unittest
-from test.support import cpython_only, requires_limited_api, skip_on_s390x
+from test.support import cpython_only, requires_limited_api, skip_on_s390x, is_wasi, Py_DEBUG
 try:
     import _testcapi
 except ImportError:
@@ -932,6 +932,7 @@ class TestErrorMessagesUseQualifiedName(unittest.TestCase):
 class TestRecursion(unittest.TestCase):
 
     @skip_on_s390x
+    @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
     def test_super_deep(self):
 
         def recurse(n):

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 42df670fe00e0ab525d89b76b3ac0a17ba782a79..ed23d31c0d3ecdbe655783ffcde083e2b7ae9f13 100644 (file)
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -607,9 +607,9 @@ class TestSpecifics(unittest.TestCase):
         # Expected limit is C_RECURSION_LIMIT * 2
         # Duplicating the limit here is a little ugly.
         # Perhaps it should be exposed somewhere...
-        fail_depth = C_RECURSION_LIMIT * 2 + 1
+        fail_depth = C_RECURSION_LIMIT + 1
         crash_depth = C_RECURSION_LIMIT * 100
-        success_depth = int(C_RECURSION_LIMIT * 1.8)
+        success_depth = int(C_RECURSION_LIMIT * 0.9)
 
         def check_limit(prefix, repeated, mode="single"):
             expect_ok = prefix + repeated * success_depth

diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py
index bf9332e40aeaf2eb8bffaf4ab59598024a9b6b6d..b3e317bd7990fd6a7b73e3179c3228c6e4c1067f 100644 (file)
--- a/Lib/test/test_isinstance.py
+++ b/Lib/test/test_isinstance.py
@@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to):
     # Make sure that calling isinstance with a deeply nested tuple for its
     # argument will raise RecursionError eventually.
     tuple_arg = (compare_to,)
-    for cnt in range(support.EXCEEDS_RECURSION_LIMIT):
+    for cnt in range(support.C_RECURSION_LIMIT * 2):
         tuple_arg = (tuple_arg,)
         fxn(arg, tuple_arg)
 

diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index 3f10f16d71996dca63f7919e3336a7d504313dd2..fa46050658afe0ec84114e41e762d72e5563ae63 100644 (file)
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -908,7 +908,7 @@ class TestBinaryPlistlib(unittest.TestCase):
         self.assertIs(b['x'], b)
 
     def test_deep_nesting(self):
-        tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000]
+        tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000]
         for N in tests:
             chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
             try:

diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index 7e16e94aa110b20a2ee83570a292572bac966e9b..196fd60d1973f61e2659eb34ae625cce1512d3da 100644 (file)
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -2965,16 +2965,18 @@ class TestExtendedArgs(unittest.TestCase):
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
     def test_trace_lots_of_globals(self):
+        count = min(1000, int(support.C_RECURSION_LIMIT * 0.8))
+
         code = """if 1:
             def f():
                 return (
                     {}
                 )
-        """.format("\n+\n".join(f"var{i}\n" for i in range(1000)))
-        ns = {f"var{i}": i for i in range(1000)}
+        """.format("\n+\n".join(f"var{i}\n" for i in range(count)))
+        ns = {f"var{i}": i for i in range(count)}
         exec(code, ns)
         counts = self.count_traces(ns["f"])
-        self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
+        self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
 
 
 class TestEdgeCases(unittest.TestCase):

diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst
new file mode 100644 (file)
index 0000000..4f6c356
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst
@@ -0,0 +1,2 @@
+Change the C recursion limits to more closely reflect the underlying
+platform limits.

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index d42c26396d5de272c9f1b077f0a37336984a3311..2c34f5c1bc26758ddf3aa950ede9abc871186683 100755 (executable)
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1393,15 +1393,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
 
     int starting_recursion_depth;
     /* Be careful here to prevent overflow. */
-    int COMPILER_STACK_FRAME_SCALE = 2;
     PyThreadState *tstate = _PyThreadState_GET();
     if (!tstate) {
         return NULL;
     }
     struct validator vstate;
-    vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    vstate.recursion_limit = C_RECURSION_LIMIT;
     int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     vstate.recursion_depth = starting_recursion_depth;
 
     PyObject *result = ast2obj_mod(state, &vstate, t);

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 6c95f07c386fd055132ce6f2add1ffbb56ce5831..ecaff2041f9565ff2c1129f3e6c1c0b15f22bd38 100644 (file)
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -13152,15 +13152,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
 
     int starting_recursion_depth;
     /* Be careful here to prevent overflow. */
-    int COMPILER_STACK_FRAME_SCALE = 2;
     PyThreadState *tstate = _PyThreadState_GET();
     if (!tstate) {
         return NULL;
     }
     struct validator vstate;
-    vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    vstate.recursion_limit = C_RECURSION_LIMIT;
     int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     vstate.recursion_depth = starting_recursion_depth;
 
     PyObject *result = ast2obj_mod(state, &vstate, t);

diff --git a/Python/ast.c b/Python/ast.c
index 82d7beec0ee510726556e207e54b80165b566fbc..76f6556ded098bf10fd3bd6af2d3f2dc41be11eb 100644 (file)
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -1038,9 +1038,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
 }
 
 
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
 int
 _PyAST_Validate(mod_ty mod)
 {
@@ -1057,9 +1054,9 @@ _PyAST_Validate(mod_ty mod)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     state.recursion_depth = starting_recursion_depth;
-    state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    state.recursion_limit = C_RECURSION_LIMIT;
 
     switch (mod->kind) {
     case Module_kind:

diff --git a/Python/ast_opt.c b/Python/ast_opt.c
index f8c4a9513236b98322f09e7b67d10dab0e61caf8..e881b7fe2d441360a7eae50fd31c6fbe80b30356 100644 (file)
--- a/Python/ast_opt.c
+++ b/Python/ast_opt.c
@@ -1102,9 +1102,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
 #undef CALL_OPT
 #undef CALL_SEQ
 
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
 int
 _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
 {
@@ -1118,9 +1115,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     state->recursion_depth = starting_recursion_depth;
-    state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    state->recursion_limit = C_RECURSION_LIMIT;
 
     int ret = astfold_mod(mod, arena, state);
     assert(ret || PyErr_Occurred());

diff --git a/Python/symtable.c b/Python/symtable.c
index 7972984a5500666ef583e0dcc077e34a3e5c9124..cfa5bec4434b3b18bbb5fd352af3e1c134bb3430 100644 (file)
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -281,11 +281,6 @@ symtable_new(void)
     return NULL;
 }
 
-/* Using a scaling factor means this should automatically adjust when
-   the recursion limit is adjusted for small or large C stack allocations.
-*/
-#define COMPILER_STACK_FRAME_SCALE 2
-
 struct symtable *
 _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
 {
@@ -312,9 +307,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     st->recursion_depth = starting_recursion_depth;
-    st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    st->recursion_limit = C_RECURSION_LIMIT;
 
     /* Make the initial symbol information gathering pass */
     if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {