GH-113655: Lower the C recursion limit on various platforms (GH-113944)

Author: markshannon

Include/cpython/pystate.h

@@ -224,10 +224,14 @@ struct _ts {
    // recursions, sometimes less. 500 is a more conservative limit.
 #  define Py_C_RECURSION_LIMIT 500
 #elif defined(__s390x__)
-#  define Py_C_RECURSION_LIMIT 1200
+#  define Py_C_RECURSION_LIMIT 800
+#elif defined(_WIN32)
+#  define Py_C_RECURSION_LIMIT 4000
+#elif defined(_Py_ADDRESS_SANITIZER)
+#  define Py_C_RECURSION_LIMIT 4000
 #else
    // This value is duplicated in Lib/test/support/__init__.py
-#  define Py_C_RECURSION_LIMIT 8000
+#  define Py_C_RECURSION_LIMIT 10000
 #endif
 
 

Lib/test/support/__init__.py

@@ -2377,7 +2377,10 @@ def _get_c_recursion_limit():
         return _testcapi.Py_C_RECURSION_LIMIT
     except (ImportError, AttributeError):
         # Originally taken from Include/cpython/pystate.h .
-        return 8000
+        if sys.platform == 'win32':
+            return 4000
+        else:
+            return 10000
 
 # The default C recursion limit.
 Py_C_RECURSION_LIMIT = _get_c_recursion_limit()

Lib/test/test_ast.py

@@ -1126,7 +1126,7 @@ def next(self):
     def test_ast_recursion_limit(self):
         fail_depth = support.EXCEEDS_RECURSION_LIMIT
         crash_depth = 100_000
-        success_depth = 1200
+        success_depth = int(support.Py_C_RECURSION_LIMIT * 0.8)
         if _testinternalcapi is not None:
             remaining = _testinternalcapi.get_c_recursion_remaining()
             success_depth = min(success_depth, remaining)

Lib/test/test_compile.py

@@ -623,12 +623,10 @@ def test_yet_more_evil_still_undecodable(self):
     @support.cpython_only
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     def test_compiler_recursion_limit(self):
-        # Expected limit is Py_C_RECURSION_LIMIT * 2
-        # Duplicating the limit here is a little ugly.
-        # Perhaps it should be exposed somewhere...
-        fail_depth = Py_C_RECURSION_LIMIT * 2 + 1
+        # Expected limit is Py_C_RECURSION_LIMIT
+        fail_depth = Py_C_RECURSION_LIMIT + 1
         crash_depth = Py_C_RECURSION_LIMIT * 100
-        success_depth = int(Py_C_RECURSION_LIMIT * 1.8)
+        success_depth = int(Py_C_RECURSION_LIMIT * 0.8)
 
         def check_limit(prefix, repeated, mode="single"):
             expect_ok = prefix + repeated * success_depth

Lib/test/test_functools.py

@@ -1875,8 +1875,14 @@ def fib(n):
             return fib(n-1) + fib(n-2)
 
         if not support.Py_DEBUG:
+            depth = support.Py_C_RECURSION_LIMIT*2//7
             with support.infinite_recursion():
-                fib(2500)
+                fib(depth)
+        if self.module == c_functools:
+            fib.cache_clear()
+            with support.infinite_recursion():
+                with self.assertRaises(RecursionError):
+                    fib(10000)
 
 
 @py_functools.lru_cache()

Lib/test/test_sys_settrace.py

@@ -3037,10 +3037,8 @@ def test_trace_unpack_long_sequence(self):
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
     def test_trace_lots_of_globals(self):
-        count = 1000
-        if _testinternalcapi is not None:
-            remaining = _testinternalcapi.get_c_recursion_remaining()
-            count = min(count, remaining)
+
+        count = min(1000, int(support.Py_C_RECURSION_LIMIT * 0.8))
 
         code = """if 1:
             def f():

Misc/NEWS.d/next/Core and Builtins/2024-01-11-01-28-25.gh-issue-113655.Mfioxp.rst

@@ -0,0 +1,3 @@
+Set the C recursion limit to 4000 on Windows, and 10000 on Linux/OSX. This
+seems to be near the sweet spot to maintain safety, but not compromise
+backwards compatibility.

Parser/asdl_c.py

@@ -1388,15 +1388,14 @@ class PartingShots(StaticVisitor):
 
     int starting_recursion_depth;
     /* Be careful here to prevent overflow. */
-    int COMPILER_STACK_FRAME_SCALE = 2;
     PyThreadState *tstate = _PyThreadState_GET();
     if (!tstate) {
         return NULL;
     }
     struct validator vstate;
-    vstate.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    vstate.recursion_limit = Py_C_RECURSION_LIMIT;
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     vstate.recursion_depth = starting_recursion_depth;
 
     PyObject *result = ast2obj_mod(state, &vstate, t);

Python/Python-ast.c

@@ -1037,10 +1037,6 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps)
     return 1;
 }
 
-
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
 int
 _PyAST_Validate(mod_ty mod)
 {
@@ -1057,9 +1053,9 @@ _PyAST_Validate(mod_ty mod)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     state.recursion_depth = starting_recursion_depth;
-    state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    state.recursion_limit = Py_C_RECURSION_LIMIT;
 
     switch (mod->kind) {
     case Module_kind:

Python/ast.c

@@ -1100,9 +1100,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat
 #undef CALL_OPT
 #undef CALL_SEQ
 
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
 int
 _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features)
 {
@@ -1120,9 +1117,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     state.recursion_depth = starting_recursion_depth;
-    state.recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    state.recursion_limit = Py_C_RECURSION_LIMIT;
 
     int ret = astfold_mod(mod, arena, &state);
     assert(ret || PyErr_Occurred());

Python/ast_opt.c

@@ -386,11 +386,6 @@ symtable_new(void)
     return NULL;
 }
 
-/* Using a scaling factor means this should automatically adjust when
-   the recursion limit is adjusted for small or large C stack allocations.
-*/
-#define COMPILER_STACK_FRAME_SCALE 2
-
 struct symtable *
 _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
 {
@@ -417,9 +412,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
     }
     /* Be careful here to prevent overflow. */
     int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
-    starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+    starting_recursion_depth = recursion_depth;
     st->recursion_depth = starting_recursion_depth;
-    st->recursion_limit = Py_C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+    st->recursion_limit = Py_C_RECURSION_LIMIT;
 
     /* Make the initial symbol information gathering pass */
     if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {

Python/symtable.c

@@ -965,7 +965,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
     unsigned int depth = 0;
     while (1) {
         if (MAX_FRAME_DEPTH <= depth) {
-            PUTS(fd, "  ...\n");
+            if (MAX_FRAME_DEPTH < depth) {
+                PUTS(fd, "plus ");
+                _Py_DumpDecimal(fd, depth);
+                PUTS(fd, " frames\n");
+            }
             break;
         }
         dump_frame(fd, frame);