gh-111798: Use lower Py_C_RECURSION_LIMIT in debug mode (#112124)

* Run again test_ast_recursion_limit() on WASI platform.
* Add _testinternalcapi.get_c_recursion_remaining().
* Fix test_ast and test_sys_settrace: test_ast_recursion_limit() and
  test_trace_unpack_long_sequence() now adjust the maximum recursion
  depth depending on the the remaining C recursion.

diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index decafde474b029d38a0880ec4f155f8674649fb9..34e23830f27e7763e9e837dca2f2a08826675231 100644 (file)
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -214,7 +214,11 @@ struct _ts {
 
 };
 
-#ifdef __wasi__
+#ifdef Py_DEBUG
+   // A debug build is likely built with low optimization level which implies
+   // higher stack memory usage than a release build: use a lower limit.
+#  define Py_C_RECURSION_LIMIT 500
+#elif defined(__wasi__)
    // WASI has limited call stack. Python's recursion limit depends on code
    // layout, optimization, and WASI runtime. Wasmtime can handle about 700
    // recursions, sometimes less. 500 is a more conservative limit.

diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 69c356ed88f57b427c6218a8d3cd38d0949d987d..64fcb02309de77eec3a958b3ae080d891d17d6b1 100644 (file)
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -12,6 +12,10 @@ import warnings
 import weakref
 from functools import partial
 from textwrap import dedent
+try:
+    import _testinternalcapi
+except ImportError:
+    _testinternalcapi = None
 
 from test import support
 from test.support.import_helper import import_fresh_module
@@ -1118,12 +1122,14 @@ class AST_Tests(unittest.TestCase):
                     return self
         enum._test_simple_enum(_Precedence, ast._Precedence)
 
-    @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     @support.cpython_only
     def test_ast_recursion_limit(self):
         fail_depth = support.EXCEEDS_RECURSION_LIMIT
         crash_depth = 100_000
         success_depth = 1200
+        if _testinternalcapi is not None:
+            remaining = _testinternalcapi.get_c_recursion_remaining()
+            success_depth = min(success_depth, remaining)
 
         def check_limit(prefix, repeated):
             expect_ok = prefix + repeated * success_depth

diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index 292096383bca6374246db1417db32222662d1969..fc5ca72236b1fb700cd4dede8052d404d4954586 100644 (file)
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -14,6 +14,10 @@ import tempfile
 import textwrap
 import subprocess
 import warnings
+try:
+    import _testinternalcapi
+except ImportError:
+    _testinternalcapi = None
 
 support.requires_working_socket(module=True)
 
@@ -3033,16 +3037,21 @@ class TestExtendedArgs(unittest.TestCase):
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
     def test_trace_lots_of_globals(self):
+        count = 1000
+        if _testinternalcapi is not None:
+            remaining = _testinternalcapi.get_c_recursion_remaining()
+            count = min(count, remaining)
+
         code = """if 1:
             def f():
                 return (
                     {}
                 )
-        """.format("\n+\n".join(f"var{i}\n" for i in range(1000)))
-        ns = {f"var{i}": i for i in range(1000)}
+        """.format("\n+\n".join(f"var{i}\n" for i in range(count)))
+        ns = {f"var{i}": i for i in range(count)}
         exec(code, ns)
         counts = self.count_traces(ns["f"])
-        self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
+        self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
 
 
 class TestEdgeCases(unittest.TestCase):

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst b/Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst
new file mode 100644 (file)
index 0000000..24bb4ec
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst
@@ -0,0 +1,4 @@
+When Python is built in debug mode, set the C recursion limit to 500 instead
+of 1500. A debug build is likely built with low optimization level which
+implies higher stack memory usage than a release build. Patch by Victor
+Stinner.

diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 604a59e7e19e43e2ffabc603627e8606aecaaa37..4fc9e853c0cd91b6a6d7a63d1c6f233179f232e9 100644 (file)
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -109,6 +109,14 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
 }
 
 
+static PyObject*
+get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args))
+{
+    PyThreadState *tstate = _PyThreadState_GET();
+    return PyLong_FromLong(tstate->c_recursion_remaining);
+}
+
+
 static PyObject*
 test_bswap(PyObject *self, PyObject *Py_UNUSED(args))
 {
@@ -1611,6 +1619,7 @@ perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args)
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
+    {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS},
     {"test_bswap", test_bswap, METH_NOARGS},
     {"test_popcount", test_popcount, METH_NOARGS},
     {"test_bit_length", test_bit_length, METH_NOARGS},