From c8357ea920866c831b1c208097a2c6a5a25e2024 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 31 Aug 2022 19:23:27 +0100 Subject: [PATCH 01/15] Raise recursion error on C stack overflow, instead of crashing. --- Include/ceval.h | 1 + Include/cpython/pystate.h | 15 ++- Include/internal/pycore_ast_state.h | 2 - Include/internal/pycore_ceval.h | 37 +++--- Include/internal/pycore_compile.h | 2 - Include/internal/pycore_pystate.h | 2 + Include/internal/pycore_symtable.h | 2 - Lib/test/list_tests.py | 2 +- Lib/test/mapping_tests.py | 2 +- Lib/test/test_ast.py | 16 ++- Lib/test/test_compile.py | 23 ++-- Lib/test/test_dict.py | 2 +- Lib/test/test_exception_group.py | 2 +- Lib/test/test_exceptions.py | 13 +-- Lib/test/test_isinstance.py | 12 +- Modules/_testinternalcapi.c | 4 +- Parser/asdl_c.py | 35 +----- Python/Python-ast.c | 93 +++------------ Python/ast.c | 53 ++------- Python/ast_opt.c | 40 +------ Python/ceval.c | 130 ++++++++++++--------- Python/errors.c | 6 +- Python/pystate.c | 77 +++++++++++- Python/symtable.c | 175 ++++++++++------------------ Python/sysmodule.c | 2 +- 25 files changed, 308 insertions(+), 440 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index ad4d909d6f2b14..fe6fa79622f803 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -74,6 +74,7 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); PyAPI_FUNC(void) Py_SetRecursionLimit(int); PyAPI_FUNC(int) Py_GetRecursionLimit(void); +PyAPI_FUNC(int) Py_StackOverflowCheck(const char *where); PyAPI_FUNC(int) Py_EnterRecursiveCall(const char *where); PyAPI_FUNC(void) Py_LeaveRecursiveCall(void); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index cc3c3eae941933..834893d1de9abc 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -95,9 +95,17 @@ struct _ts { /* Was this thread state statically allocated? */ int _static; - int recursion_remaining; + /* Python recursion limit handling */ + int py_recursion_remaining; int recursion_limit; - int recursion_headroom; /* Allow 50 more calls to handle any errors. */ + int py_recursion_headroom; /* Allow 50 more calls to handle any errors. */ + + /* C stack overflow handling */ + intptr_t stack_limit; + intptr_t yellow_stack_limit; + intptr_t red_stack_limit; + int stack_in_yellow; + int stack_grows; /* 'tracing' keeps track of the execution depth when tracing/profiling. This is to prevent the actual trace/profile code from being recorded in @@ -367,3 +375,6 @@ typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + + +PyAPI_FUNC(int) _Py_OS_GetStackLimits(void** low, void** high); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f15b4905eed14b..da78bba3b69bdf 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -12,8 +12,6 @@ extern "C" { struct ast_state { int initialized; - int recursion_depth; - int recursion_limit; PyObject *AST_type; PyObject *Add_singleton; PyObject *Add_type; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 4914948c6ca744..c238cca4a2e9bf 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -114,40 +114,39 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -#ifdef USE_STACKCHECK -/* With USE_STACKCHECK macro defined, trigger stack checks in - _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->recursion_remaining-- <= 0 - || (tstate->recursion_remaining & 63) == 0); -} -#else -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->recursion_remaining-- <= 0; -} -#endif - PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); -static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, - const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + +PyAPI_FUNC(int) _Py_StackOverflowCheckCall( + PyThreadState *tstate, + const char *where, + intptr_t scaled_location); + +static inline int +_Py_StackOverflowCheck(PyThreadState *tstate, const char *where) +{ + char addr; + intptr_t here = ((uintptr_t)&addr)/SIZEOF_VOID_P; + intptr_t here_upward = here*tstate->stack_grows; + if (here_upward < tstate->stack_limit) { + return 0; + } + return _Py_StackOverflowCheckCall(tstate, where, here_upward); } +#define _Py_EnterRecursiveCallTstate _Py_StackOverflowCheck + static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCallTstate(tstate, where); } static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { - tstate->recursion_remaining++; } static inline void _Py_LeaveRecursiveCall(void) { - PyThreadState *tstate = _PyThreadState_GET(); - _Py_LeaveRecursiveCallTstate(tstate); } extern struct _PyInterpreterFrame* _PyEval_GetFrame(void); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1a628a08ca4ebf..4de55eb2cb70c8 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -29,8 +29,6 @@ typedef struct { int optimize; int ff_features; - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ } _PyASTOptimizeState; extern int _PyAST_Optimize( diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 3d6d400f74dd1d..cdeb8ff7c806d0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -162,6 +162,8 @@ PyAPI_FUNC(int) _PyState_AddModule( PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); +int _Py_UpdateStackLimits(PyThreadState *tstate); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 2d64aba22ff905..3d9bf66bfbc91c 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -37,8 +37,6 @@ struct symtable { PyObject *st_private; /* name of current class or NULL */ PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ }; typedef struct _symtable_entry { diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461f..dd97442adb01d9 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(100_000): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 613206a0855aea..b7f9ce9bbee1bd 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -624,7 +624,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(100_000): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 4cfefe4ac3dd1d..ed6935a04886a3 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -816,19 +816,17 @@ def next(self): @support.cpython_only def test_ast_recursion_limit(self): - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + crash_depth = 100_000 + success_depth = 5_000 def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth ast.parse(expect_ok) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - ast.parse(broken) + broken = prefix + repeated * crash_depth + details = "Compiling ({!r} + {!r} * {})".format( + prefix, repeated, crash_depth) + with self.assertRaises(RecursionError, msg=details): + ast.parse(broken) check_limit("a", "()") check_limit("a", ".b") diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d7b78f686ef88d..1683389db52373 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -546,26 +546,17 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + crash_depth = 100_000 + success_depth = 5_000 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth compile(expect_ok, '', mode) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - compile(broken, '', mode) + broken = prefix + repeated * crash_depth + details = "Compiling ({!r} + {!r} * {})".format( + prefix, repeated, crash_depth) + with self.assertRaises(RecursionError, msg=details): + compile(broken, '', mode) check_limit("a", "()") check_limit("a", ".b") diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 5b8baaf9e6e280..e6c69249de33d5 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(100_000): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index aa28e16bedfa68..08f715fb4aea89 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -405,7 +405,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(100_000): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index f8b05128e3f8b9..36b9774bd12e1d 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1375,16 +1375,6 @@ def test_recursion_normalizing_exception(self): class MyException(Exception): pass - def setrecursionlimit(depth): - while 1: - try: - sys.setrecursionlimit(depth) - return depth - except RecursionError: - # sys.setrecursionlimit() raises a RecursionError if - # the new recursion limit is too low (issue #25274). - depth += 1 - def recurse(cnt): cnt -= 1 if cnt: @@ -1405,9 +1395,8 @@ def gen(): # tstate->recursion_depth is equal to (recursion_limit - 1) # and is equal to recursion_limit when _gen_throw() calls # PyErr_NormalizeException(). - recurse(setrecursionlimit(depth + 2) - depth) + recurse(1000) finally: - sys.setrecursionlimit(recursionlimit) print('Done.') """ % __file__ rc, out, err = script_helper.assert_python_failure("-Wd", "-c", code) diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index a0974640bc1146..d2980b40418d5c 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -8,7 +8,7 @@ from test import support - + class TestIsInstanceExceptions(unittest.TestCase): # Test to make sure that an AttributeError when accessing the instance's # class's bases is masked. This was actually a bug in Python 2.2 and @@ -97,7 +97,7 @@ def getclass(self): class D: pass self.assertRaises(RuntimeError, isinstance, c, D) - + # These tests are similar to above, but tickle certain code paths in # issubclass() instead of isinstance() -- really PyObject_IsSubclass() # vs. PyObject_IsInstance(). @@ -147,7 +147,7 @@ def getbases(self): self.assertRaises(TypeError, issubclass, B, C()) - + # meta classes for creating abstract classes and instances class AbstractClass(object): def __init__(self, bases): @@ -179,7 +179,7 @@ class Super: class Child(Super): pass - + class TestIsInstanceIsSubclass(unittest.TestCase): # Tests to ensure that isinstance and issubclass work on abstract # classes and instances. Before the 2.2 release, TypeErrors were @@ -353,10 +353,10 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(sys.getrecursionlimit()+5): + for cnt in range(100_000): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) - + if __name__ == '__main__': unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 02a061b84f85a3..c8379c6ffbc093 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -44,9 +44,7 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args)) { PyThreadState *tstate = _PyThreadState_GET(); - /* subtract one to ignore the frame of the get_recursion_depth() call */ - - return PyLong_FromLong(tstate->recursion_limit - tstate->recursion_remaining - 1); + return PyLong_FromLong(tstate->recursion_limit - tstate->py_recursion_remaining); } diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 13dd44ca0cdc3f..631f7c2bc159eb 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1112,8 +1112,6 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' - state->recursion_depth = 0; - state->recursion_limit = 0; state->initialized = 1; return 1; } @@ -1261,14 +1259,11 @@ def func_begin(self, name): self.emit('if (!o) {', 1) self.emit("Py_RETURN_NONE;", 2) self.emit("}", 1) - self.emit("if (++state->recursion_depth > state->recursion_limit) {", 1) - self.emit("PyErr_SetString(PyExc_RecursionError,", 2) - self.emit('"maximum recursion depth exceeded during ast construction");', 3) + self.emit('if (Py_StackOverflowCheck("during ast construction")) {', 1) self.emit("return 0;", 2) self.emit("}", 1) def func_end(self): - self.emit("state->recursion_depth--;", 1) self.emit("return result;", 1) self.emit("failed:", 0) self.emit("Py_XDECREF(value);", 1) @@ -1380,31 +1375,7 @@ class PartingShots(StaticVisitor): return NULL; } - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; - /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; - PyThreadState *tstate = _PyThreadState_GET(); - if (!tstate) { - return 0; - } - state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state->recursion_depth = starting_recursion_depth; - - PyObject *result = ast2obj_mod(state, t); - - /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; - } - return result; + return ast2obj_mod(state, t); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ @@ -1470,8 +1441,6 @@ def visit(self, object): def generate_ast_state(module_state, f): f.write('struct ast_state {\n') f.write(' int initialized;\n') - f.write(' int recursion_depth;\n') - f.write(' int recursion_limit;\n') for s in module_state: f.write(' PyObject *' + s + ';\n') f.write('};') diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f485af675ccff7..a01b7918b209dc 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1851,8 +1851,6 @@ init_types(struct ast_state *state) "TypeIgnore(int lineno, string tag)"); if (!state->TypeIgnore_type) return 0; - state->recursion_depth = 0; - state->recursion_limit = 0; state->initialized = 1; return 1; } @@ -3612,9 +3610,7 @@ ast2obj_mod(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } switch (o->kind) { @@ -3672,7 +3668,6 @@ ast2obj_mod(struct ast_state *state, void* _o) Py_DECREF(value); break; } - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -3689,9 +3684,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } switch (o->kind) { @@ -4237,7 +4230,6 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -4254,9 +4246,7 @@ ast2obj_expr(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } switch (o->kind) { @@ -4720,7 +4710,6 @@ ast2obj_expr(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -4863,9 +4852,7 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->comprehension_type; @@ -4891,7 +4878,6 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->is_async, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -4908,9 +4894,7 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } switch (o->kind) { @@ -4956,7 +4940,6 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -4973,9 +4956,7 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->arguments_type; @@ -5016,7 +4997,6 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->defaults, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5033,9 +5013,7 @@ ast2obj_arg(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->arg_type; @@ -5076,7 +5054,6 @@ ast2obj_arg(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5093,9 +5070,7 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->keyword_type; @@ -5131,7 +5106,6 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5148,9 +5122,7 @@ ast2obj_alias(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->alias_type; @@ -5186,7 +5158,6 @@ ast2obj_alias(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5203,9 +5174,7 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->withitem_type; @@ -5221,7 +5190,6 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->optional_vars, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5238,9 +5206,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } tp = (PyTypeObject *)state->match_case_type; @@ -5261,7 +5227,6 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5278,9 +5243,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } switch (o->kind) { @@ -5422,7 +5385,6 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -5439,9 +5401,7 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (Py_StackOverflowCheck("during ast construction")) { return 0; } switch (o->kind) { @@ -5461,7 +5421,6 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) Py_DECREF(value); break; } - state->recursion_depth--; return result; failed: Py_XDECREF(value); @@ -12315,31 +12274,7 @@ PyObject* PyAST_mod2obj(mod_ty t) return NULL; } - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; - /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; - PyThreadState *tstate = _PyThreadState_GET(); - if (!tstate) { - return 0; - } - state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state->recursion_depth = starting_recursion_depth; - - PyObject *result = ast2obj_mod(state, t); - - /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; - } - return result; + return ast2obj_mod(state, t); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ diff --git a/Python/ast.c b/Python/ast.c index a0321b58ba8cff..81dfc0be8a8fba 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4,14 +4,14 @@ */ #include "Python.h" #include "pycore_ast.h" // asdl_stmt_seq +#include "pycore_ceval.h" // _Py_StackOverflowCheck #include "pycore_pystate.h" // _PyThreadState_GET() #include #include struct validator { - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ + int placeholder; }; static int validate_stmts(struct validator *, asdl_stmt_seq *); @@ -161,9 +161,7 @@ validate_constant(struct validator *state, PyObject *value) return 1; if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) { - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (Py_StackOverflowCheck("during compilation")) { return 0; } @@ -190,7 +188,6 @@ validate_constant(struct validator *state, PyObject *value) } Py_DECREF(it); - --state->recursion_depth; return 1; } @@ -207,9 +204,8 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) { VALIDATE_POSITIONS(exp); int ret = -1; - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + + if (Py_StackOverflowCheck("during compilation")) { return 0; } int check_ctx = 1; @@ -387,7 +383,6 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) PyErr_SetString(PyExc_SystemError, "unexpected expression"); ret = 0; } - state->recursion_depth--; return ret; } @@ -530,9 +525,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) { VALIDATE_POSITIONS(p); int ret = -1; - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (Py_StackOverflowCheck("during compilation")) { return 0; } switch (p->kind) { @@ -668,7 +661,6 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) PyErr_SetString(PyExc_SystemError, "unexpected pattern"); ret = 0; } - state->recursion_depth--; return ret; } @@ -701,9 +693,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) VALIDATE_POSITIONS(stmt); int ret = -1; Py_ssize_t i; - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (Py_StackOverflowCheck("during compilation")) { return 0; } switch (stmt->kind) { @@ -909,7 +899,6 @@ validate_stmt(struct validator *state, stmt_ty stmt) PyErr_SetString(PyExc_SystemError, "unexpected statement"); ret = 0; } - state->recursion_depth--; return ret; } @@ -965,32 +954,16 @@ validate_patterns(struct validator *state, asdl_pattern_seq *patterns, int star_ return 1; } - -/* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 - int _PyAST_Validate(mod_ty mod) { int res = -1; struct validator state; - PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; - /* Setup recursion depth check counters */ - tstate = _PyThreadState_GET(); - if (!tstate) { + /* Be careful here to prevent overflow. */ + if (Py_StackOverflowCheck("_PyAST_Validate")) { return 0; } - /* Be careful here to prevent overflow. */ - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth< INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state.recursion_depth = starting_recursion_depth; - state.recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - switch (mod->kind) { case Module_kind: res = validate_stmts(&state, mod->v.Module.body); @@ -1012,14 +985,6 @@ _PyAST_Validate(mod_ty mod) PyErr_SetString(PyExc_SystemError, "impossible module node"); return 0; } - - /* Check that the recursion depth counting balanced correctly */ - if (res && state.recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST validator recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state.recursion_depth); - return 0; - } return res; } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index b1d807bcf10ae1..5f429dc665e498 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -705,9 +705,7 @@ astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (Py_StackOverflowCheck("during compilation")) { return 0; } switch (node_->kind) { @@ -808,7 +806,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case Name_kind: if (node_->v.Name.ctx == Load && _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { - state->recursion_depth--; return make_const(node_, PyBool_FromLong(!state->optimize), ctx_); } break; @@ -821,7 +818,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new expression // kinds are added without being handled here } - state->recursion_depth--; return 1; } @@ -868,9 +864,7 @@ astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (Py_StackOverflowCheck("during compilation")) { return 0; } switch (node_->kind) { @@ -988,7 +982,6 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new statement // kinds are added without being handled here } - state->recursion_depth--; return 1; } @@ -1020,9 +1013,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // Currently, this is really only used to form complex/negative numeric // constants in MatchValue and MatchMapping nodes // We still recurse into all subexpressions and subpatterns anyway - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (Py_StackOverflowCheck("during compilation")) { return 0; } switch (node_->kind) { @@ -1056,7 +1047,6 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new pattern // kinds are added without being handled here } - state->recursion_depth--; return 1; } @@ -1079,33 +1069,9 @@ astfold_match_case(match_case_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat int _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) { - PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; - - /* Setup recursion depth check counters */ - tstate = _PyThreadState_GET(); - if (!tstate) { - return 0; - } - /* Be careful here to prevent overflow. */ - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state->recursion_depth = starting_recursion_depth; - state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - int ret = astfold_mod(mod, arena, state); assert(ret || PyErr_Occurred()); - /* Check that the recursion depth counting balanced correctly */ - if (ret && state->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST optimizer recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; - } return ret; } diff --git a/Python/ceval.c b/Python/ceval.c index c61ccd7dfc6f56..51d5f88575f66c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -257,53 +257,86 @@ Py_SetRecursionLimit(int new_limit) PyInterpreterState *interp = _PyInterpreterState_GET(); interp->ceval.recursion_limit = new_limit; for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { - int depth = p->recursion_limit - p->recursion_remaining; + int depth = p->recursion_limit - p->py_recursion_remaining; p->recursion_limit = new_limit; - p->recursion_remaining = new_limit - depth; + p->py_recursion_remaining = new_limit - depth; } } +int +_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) +{ + return _Py_StackOverflowCheck(tstate, where); +} + +int +_Py_StackOverflowCheckCall(PyThreadState *tstate, const char *where, intptr_t stack_location) +{ + assert(stack_location >= tstate->stack_limit); + assert(tstate->stack_grows); + if (stack_location >= tstate->yellow_stack_limit) { + if (_Py_UpdateStackLimits(tstate)) { + return -1; + } + } + if (stack_location < tstate->yellow_stack_limit) { + tstate->stack_limit = tstate->yellow_stack_limit; + tstate->stack_in_yellow = false; + return 0; + } + if (stack_location >= tstate->red_stack_limit) { + printf("Relative stack limits: %ld %ld %ld\n", + tstate->stack_limit - stack_location, + tstate->yellow_stack_limit - stack_location, + tstate->red_stack_limit - stack_location); + Py_FatalError("Unrecoverable C stack overflow"); + } + /* In yellow zone */ + assert(stack_location >= tstate->yellow_stack_limit && + stack_location < tstate->red_stack_limit); + if (!tstate->stack_in_yellow) { + tstate->stack_limit = INTPTR_MIN; + tstate->stack_in_yellow = true; + _PyErr_Format(tstate, PyExc_RecursionError, + "C stack overflow %s", + where); + return -1; + } + assert(tstate->stack_limit == INTPTR_MIN); + return 0; +} + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ int -_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) +_Py_CheckPyRecursiveCall(PyThreadState *tstate) { /* Check against global limit first. */ - int depth = tstate->recursion_limit - tstate->recursion_remaining; + int depth = tstate->recursion_limit - tstate->py_recursion_remaining; if (depth < tstate->interp->ceval.recursion_limit) { tstate->recursion_limit = tstate->interp->ceval.recursion_limit; - tstate->recursion_remaining = tstate->recursion_limit - depth; - assert(tstate->recursion_remaining > 0); + tstate->py_recursion_remaining = tstate->recursion_limit - depth; + assert(tstate->py_recursion_remaining > 0); return 0; } -#ifdef USE_STACKCHECK - if (PyOS_CheckStack()) { - ++tstate->recursion_remaining; - _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow"); - return -1; - } -#endif - if (tstate->recursion_headroom) { - if (tstate->recursion_remaining < -50) { + if (tstate->py_recursion_headroom) { + if (tstate->py_recursion_remaining < -50) { /* Overflowing while handling an overflow. Give up. */ Py_FatalError("Cannot recover from stack overflow."); } } else { - if (tstate->recursion_remaining <= 0) { - tstate->recursion_headroom++; + if (tstate->py_recursion_remaining <= 0) { + tstate->py_recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, - "maximum recursion depth exceeded%s", - where); - tstate->recursion_headroom--; - ++tstate->recursion_remaining; + "maximum recursion depth exceeded"); + tstate->py_recursion_headroom--; return -1; } } return 0; } - static const binaryfunc binary_ops[] = { [NB_ADD] = PyNumber_Add, [NB_AND] = PyNumber_And, @@ -1036,14 +1069,16 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->previous = prev_cframe->current_frame; cframe.current_frame = frame; + if (_Py_EnterRecursiveCallTstate(tstate, "calling a Python function")) { + tstate->py_recursion_remaining--; + goto exit_unwind; + } + /* support for generator.throw() */ if (throwflag) { - if (_Py_EnterRecursiveCallTstate(tstate, "")) { - tstate->recursion_remaining--; - goto exit_unwind; - } TRACE_FUNCTION_THROW_ENTRY(); DTRACE_FUNCTION_ENTRY(); + tstate->py_recursion_remaining--; goto resume_with_error; } @@ -1078,9 +1113,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int start_frame: - if (_Py_EnterRecursiveCallTstate(tstate, "")) { - tstate->recursion_remaining--; - goto exit_unwind; + tstate->py_recursion_remaining--; + if (tstate->py_recursion_remaining < 0) { + if (_Py_CheckPyRecursiveCall(tstate)) { + goto exit_unwind; + } } resume_frame: @@ -1830,7 +1867,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; if (!frame->is_entry) { frame = cframe.current_frame = pop_frame(tstate, frame); _PyFrame_StackPush(frame, retval); @@ -2046,7 +2083,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; tstate->cframe->use_tracing = cframe.use_tracing; @@ -4493,14 +4530,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(CALL, hit); JUMPBY(INLINE_CACHE_ENTRIES_CALL); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } PyObject *arg = TOP(); PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); @@ -4695,13 +4726,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(CALL, hit); JUMPBY(INLINE_CACHE_ENTRIES_CALL); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } PyObject *res = cfunc(self, arg); - _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); @@ -4767,13 +4792,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(CALL, hit); JUMPBY(INLINE_CACHE_ENTRIES_CALL); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } PyObject *res = cfunc(self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); STACK_SHRINK(oparg + 1); @@ -4915,7 +4934,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; if (!frame->is_entry) { _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); @@ -5287,7 +5306,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int exit_unwind: assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; if (frame->is_entry) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; @@ -5858,11 +5877,9 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + frame->f_code->co_framesize == tstate->datastack_top); - tstate->recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); assert(frame->owner == FRAME_OWNED_BY_THREAD); _PyFrame_Clear(frame); - tstate->recursion_remaining++; _PyThreadState_PopFrame(tstate, frame); } @@ -7456,3 +7473,10 @@ void Py_LeaveRecursiveCall(void) { _Py_LeaveRecursiveCall(); } + + +int Py_StackOverflowCheck(const char *where) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _Py_StackOverflowCheck(tstate, where); +} diff --git a/Python/errors.c b/Python/errors.c index 2aa748c60c3704..a3007531e412c9 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -310,14 +310,14 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc, PyObject **val, PyObject **tb) { int recursion_depth = 0; - tstate->recursion_headroom++; + tstate->py_recursion_headroom++; PyObject *type, *value, *initial_tb; restart: type = *exc; if (type == NULL) { /* There was no exception, so nothing to do. */ - tstate->recursion_headroom--; + tstate->py_recursion_headroom--; return; } @@ -369,7 +369,7 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc, } *exc = type; *val = value; - tstate->recursion_headroom--; + tstate->py_recursion_headroom--; return; error: diff --git a/Python/pystate.c b/Python/pystate.c index a11f1622ecd4ab..2534a48c229442 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -763,6 +763,74 @@ free_threadstate(PyThreadState *tstate) the thread is getting added to the interpreter. */ +Py_NO_INLINE +static uintptr_t +callee_address(void) + { + char addr; + uintptr_t here = (uintptr_t)&addr; + return here; +} + +static int +stack_grows(void) { + char addr; + uintptr_t here = (uintptr_t)&addr; + uintptr_t there = callee_address(); + if (there > here) { + /* Stack grows up */ + return 1; + } + else { + return -1; + } +} +#define DEFAULT_STACK_ALLOWANCE (1<<16) +#define SIZE_OF_RED_ZONE (1<<10) +#define SIZE_OF_YELLOW_ZONE (3<<10) + +int +_Py_OS_GetStackLimits(void** low, void** high); + +int +_Py_UpdateStackLimits(PyThreadState *tstate) +{ +#ifdef HAVE_OS_STACK_LIMITS + assert(tstate->stack_grows); + void *lptr, *hptr; + if (_Py_Update_StackLimits(tstate, &lptr, &hptr)) { + return -1; + } + intptr_t low = (intptr_t)lptr; + intptr_t high = (intptr_t)hptr; + low = ((uintptr_t)low)/SIZEOF_VOID_P; + high = ((uintptr_t)high/SIZEOF_VOID_P; + low *= tstate->stack_grows; + high *= tstate->stack_grows; + if (low > high) { + intptr_t temp = low; + low = high; + high = temp; + } + tstate->red_stack_limit = high - SIZE_OF_RED_ZONE; + tstate->yellow_stack_limit = tstate->red_stack_limit - SIZE_OF_YELLOW_ZONE; +#else + char addr; + intptr_t here = ((uintptr_t)&addr)/SIZEOF_VOID_P; + here *= tstate->stack_grows; + if (here < tstate->red_stack_limit - DEFAULT_STACK_ALLOWANCE || + here > tstate->red_stack_limit + DEFAULT_STACK_ALLOWANCE/2) + { + /* Either uninitialized or + * sufficiently out of bounds, that we must have the wrong thread. */ + intptr_t high = here + DEFAULT_STACK_ALLOWANCE; + tstate->red_stack_limit = high - SIZE_OF_RED_ZONE; + tstate->yellow_stack_limit = tstate->red_stack_limit - SIZE_OF_YELLOW_ZONE; + } +#endif + return 0; +} + static void init_threadstate(PyThreadState *tstate, PyInterpreterState *interp, uint64_t id, @@ -793,7 +861,7 @@ init_threadstate(PyThreadState *tstate, #endif tstate->recursion_limit = interp->ceval.recursion_limit, - tstate->recursion_remaining = interp->ceval.recursion_limit, + tstate->py_recursion_remaining = interp->ceval.recursion_limit, tstate->exc_info = &tstate->exc_state; @@ -802,6 +870,13 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; + tstate->stack_grows = stack_grows(); + + tstate->stack_limit = INTPTR_MIN; + tstate->yellow_stack_limit = INTPTR_MIN; + tstate->red_stack_limit = INTPTR_MIN; + tstate->stack_in_yellow = 0; + tstate->_initialized = 1; } diff --git a/Python/symtable.c b/Python/symtable.c index 0b259b08b61f97..8b7f82c76dea70 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -259,27 +259,12 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when - the recursion limit is adjusted for small or large C stack allocations. -*/ -#define COMPILER_STACK_FRAME_SCALE 3 - struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) { struct symtable *st = symtable_new(); asdl_stmt_seq *seq; int i; - PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; if (st == NULL) return NULL; @@ -291,20 +276,6 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) st->st_filename = filename; st->st_future = future; - /* Setup recursion depth check counters */ - tstate = _PyThreadState_GET(); - if (!tstate) { - _PySymtable_Free(st); - return NULL; - } - /* Be careful here to prevent overflow. */ - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - st->recursion_depth = starting_recursion_depth; - st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { _PySymtable_Free(st); @@ -340,14 +311,6 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) _PySymtable_Free(st); return NULL; } - /* Check that the recursion depth counting balanced correctly */ - if (st->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "symtable analysis recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, st->recursion_depth); - _PySymtable_Free(st); - return NULL; - } /* Make the second symbol analysis pass */ if (symtable_analyze(st)) return st; @@ -1119,17 +1082,11 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is useful if the first node in the sequence requires special treatment. - - VISIT_QUIT macro returns the specified value exiting from the function but - first adjusts current recursion counter depth. */ -#define VISIT_QUIT(ST, X) \ - return --(ST)->recursion_depth,(X) - #define VISIT(ST, TYPE, V) \ if (!symtable_visit_ ## TYPE((ST), (V))) \ - VISIT_QUIT((ST), 0); + return 0; #define VISIT_SEQ(ST, TYPE, SEQ) { \ int i; \ @@ -1137,7 +1094,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, for (i = 0; i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ + return 0; \ } \ } @@ -1147,7 +1104,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, for (i = (START); i < asdl_seq_LEN(seq); i++) { \ TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ + return 0; \ } \ } @@ -1158,7 +1115,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \ if (!elt) continue; /* can be NULL */ \ if (!symtable_visit_ ## TYPE((ST), elt)) \ - VISIT_QUIT((ST), 0); \ + return 0; \ } \ } @@ -1188,37 +1145,35 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno, static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (Py_StackOverflowCheck("during compilation")) { + return 0; } switch (s->kind) { case FunctionDef_kind: if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (s->v.FunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); if (s->v.FunctionDef.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults); if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, s->v.FunctionDef.returns)) - VISIT_QUIT(st, 0); + return 0; if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); if (!symtable_enter_block(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; VISIT(st, arguments, s->v.FunctionDef.args); VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; break; case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; VISIT_SEQ(st, expr, s->v.ClassDef.bases); VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); if (s->v.ClassDef.decorator_list) @@ -1226,13 +1181,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, (void *)s, s->lineno, s->col_offset, s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); + return 0; tmp = st->st_private; st->st_private = s->v.ClassDef.name; VISIT_SEQ(st, stmt, s->v.ClassDef.body); st->st_private = tmp; if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; break; } case Return_kind: @@ -1253,7 +1208,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) expr_ty e_name = s->v.AnnAssign.target; long cur = symtable_lookup(st, e_name->v.Name.id); if (cur < 0) { - VISIT_QUIT(st, 0); + return 0; } if ((cur & (DEF_GLOBAL | DEF_NONLOCAL)) && (st->st_cur->ste_symbols != st->st_global) @@ -1266,17 +1221,17 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->col_offset + 1, s->end_lineno, s->end_col_offset + 1); - VISIT_QUIT(st, 0); + return 0; } if (s->v.AnnAssign.simple && !symtable_add_def(st, e_name->v.Name.id, DEF_ANNOT | DEF_LOCAL, LOCATION(e_name))) { - VISIT_QUIT(st, 0); + return 0; } else { if (s->v.AnnAssign.value && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL, LOCATION(e_name))) { - VISIT_QUIT(st, 0); + return 0; } } } @@ -1284,7 +1239,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.AnnAssign.target); } if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { - VISIT_QUIT(st, 0); + return 0; } if (s->v.AnnAssign.value) { @@ -1357,7 +1312,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) identifier name = (identifier)asdl_seq_GET(seq, i); long cur = symtable_lookup(st, name); if (cur < 0) - VISIT_QUIT(st, 0); + return 0; if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) { const char* msg; if (cur & DEF_PARAM) { @@ -1376,13 +1331,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->col_offset + 1, s->end_lineno, s->end_col_offset + 1); - VISIT_QUIT(st, 0); + return 0; } if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (!symtable_record_directive(st, name, s->lineno, s->col_offset, s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); + return 0; } break; } @@ -1393,7 +1348,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) identifier name = (identifier)asdl_seq_GET(seq, i); long cur = symtable_lookup(st, name); if (cur < 0) - VISIT_QUIT(st, 0); + return 0; if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) { const char* msg; if (cur & DEF_PARAM) { @@ -1411,13 +1366,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->col_offset + 1, s->end_lineno, s->end_col_offset + 1); - VISIT_QUIT(st, 0); + return 0; } if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (!symtable_record_directive(st, name, s->lineno, s->col_offset, s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); + return 0; } break; } @@ -1435,7 +1390,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case AsyncFunctionDef_kind: if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) - VISIT_QUIT(st, 0); + return 0; if (s->v.AsyncFunctionDef.args->defaults) VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults); if (s->v.AsyncFunctionDef.args->kw_defaults) @@ -1443,19 +1398,19 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.AsyncFunctionDef.args->kw_defaults); if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, s->v.AsyncFunctionDef.returns)) - VISIT_QUIT(st, 0); + return 0; if (s->v.AsyncFunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list); if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, FunctionBlock, (void *)s, s->lineno, s->col_offset, s->end_lineno, s->end_col_offset)) - VISIT_QUIT(st, 0); + return 0; st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; break; case AsyncWith_kind: VISIT_SEQ(st, withitem, s->v.AsyncWith.items); @@ -1469,7 +1424,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse); break; } - VISIT_QUIT(st, 1); + return 1; } static int @@ -1500,7 +1455,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) e->col_offset + 1, e->end_lineno, e->end_col_offset + 1); - VISIT_QUIT(st, 0); + return 0; } continue; } @@ -1510,22 +1465,22 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) long target_in_scope = _PyST_GetSymbol(ste, target_name); if (target_in_scope & DEF_GLOBAL) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; } else { if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; } if (!symtable_record_directive(st, target_name, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e)); } /* If we find a ModuleBlock entry, add as GLOBAL */ if (ste->ste_type == ModuleBlock) { if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; if (!symtable_record_directive(st, target_name, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e)); } @@ -1537,7 +1492,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) e->col_offset + 1, e->end_lineno, e->end_col_offset + 1); - VISIT_QUIT(st, 0); + return 0; } } @@ -1574,18 +1529,16 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) static int symtable_visit_expr(struct symtable *st, expr_ty e) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (Py_StackOverflowCheck("during compilation")) { + return 0; } switch (e->kind) { case NamedExpr_kind: if (!symtable_raise_if_annotation_block(st, "named expression", e)) { - VISIT_QUIT(st, 0); + return 0; } if(!symtable_handle_namedexpr(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case BoolOp_kind: VISIT_SEQ(st, expr, e->v.BoolOp.values); @@ -1606,11 +1559,11 @@ symtable_visit_expr(struct symtable *st, expr_ty e) FunctionBlock, (void *)e, e->lineno, e->col_offset, e->end_lineno, e->end_col_offset)) - VISIT_QUIT(st, 0); + return 0; VISIT(st, arguments, e->v.Lambda.args); VISIT(st, expr, e->v.Lambda.body); if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + return 0; break; } case IfExp_kind: @@ -1627,23 +1580,23 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case GeneratorExp_kind: if (!symtable_visit_genexp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case ListComp_kind: if (!symtable_visit_listcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case SetComp_kind: if (!symtable_visit_setcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case DictComp_kind: if (!symtable_visit_dictcomp(st, e)) - VISIT_QUIT(st, 0); + return 0; break; case Yield_kind: if (!symtable_raise_if_annotation_block(st, "yield expression", e)) { - VISIT_QUIT(st, 0); + return 0; } if (e->v.Yield.value) VISIT(st, expr, e->v.Yield.value); @@ -1654,7 +1607,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case YieldFrom_kind: if (!symtable_raise_if_annotation_block(st, "yield expression", e)) { - VISIT_QUIT(st, 0); + return 0; } VISIT(st, expr, e->v.YieldFrom.value); st->st_cur->ste_generator = 1; @@ -1664,7 +1617,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; case Await_kind: if (!symtable_raise_if_annotation_block(st, "await expression", e)) { - VISIT_QUIT(st, 0); + return 0; } VISIT(st, expr, e->v.Await.value); st->st_cur->ste_coroutine = 1; @@ -1711,13 +1664,13 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Name_kind: if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && st->st_cur->ste_type == FunctionBlock && _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e))) - VISIT_QUIT(st, 0); + return 0; } break; /* child nodes of List and Tuple will have expr_context set */ @@ -1728,16 +1681,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Tuple.elts); break; } - VISIT_QUIT(st, 1); + return 1; } static int symtable_visit_pattern(struct symtable *st, pattern_ty p) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (Py_StackOverflowCheck("during compilation")) { + return 0; } switch (p->kind) { case MatchValue_kind: @@ -1778,7 +1729,7 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT_SEQ(st, pattern, p->v.MatchOr.patterns); break; } - VISIT_QUIT(st, 1); + return 1; } static int @@ -1821,11 +1772,11 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation) (void *)annotation, annotation->lineno, annotation->col_offset, annotation->end_lineno, annotation->end_col_offset)) { - VISIT_QUIT(st, 0); + return 0; } VISIT(st, expr, annotation); if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + return 0; } return 1; } @@ -1855,7 +1806,7 @@ symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, (void *)o, o->lineno, o->col_offset, o->end_lineno, o->end_col_offset)) { - VISIT_QUIT(st, 0); + return 0; } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; @@ -1868,10 +1819,10 @@ symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + return 0; } if (returns && !symtable_visit_annotation(st, returns)) { - VISIT_QUIT(st, 0); + return 0; } return 1; } @@ -2127,7 +2078,7 @@ symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) { PyErr_RangedSyntaxLocationObject(st->st_filename, e->lineno, e->col_offset + 1, e->end_lineno, e->end_col_offset + 1); - VISIT_QUIT(st, 0); + return 0; } struct symtable * diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 75e64553d88c9f..495588514a7f3a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1217,7 +1217,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit) /* Reject too low new limit if the current recursion depth is higher than the new low-water mark. */ - int depth = tstate->recursion_limit - tstate->recursion_remaining; + int depth = tstate->recursion_limit - tstate->py_recursion_remaining; if (depth >= new_limit) { _PyErr_Format(tstate, PyExc_RecursionError, "cannot set the recursion limit to %i at " From b7b5ca5f3194c478950d0d55ed89f312076419f5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 31 Aug 2022 19:35:25 +0100 Subject: [PATCH 02/15] Add news. --- .../2022-08-31-19-35-19.gh-issue-91079.K0A-QO.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-08-31-19-35-19.gh-issue-91079.K0A-QO.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-08-31-19-35-19.gh-issue-91079.K0A-QO.rst b/Misc/NEWS.d/next/Core and Builtins/2022-08-31-19-35-19.gh-issue-91079.K0A-QO.rst new file mode 100644 index 00000000000000..314f6ef9d491c1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-08-31-19-35-19.gh-issue-91079.K0A-QO.rst @@ -0,0 +1,3 @@ +Recursion that would have overflowed the C stack is prevented and a +RecursionError raised. Prevents VM crashes when the recursion limit is set +too high. From 013f6eb94e854443a96507b6ecc112afb6bce0df Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 31 Aug 2022 19:50:01 +0100 Subject: [PATCH 03/15] Don't add Py_StackOverflowCheck to the public API (for now). --- Include/ceval.h | 1 - Include/internal/pycore_ceval.h | 7 ++++++- Python/ast_opt.c | 1 + Python/ceval.c | 10 ++-------- Python/symtable.c | 1 + 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index fe6fa79622f803..ad4d909d6f2b14 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -74,7 +74,6 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); PyAPI_FUNC(void) Py_SetRecursionLimit(int); PyAPI_FUNC(int) Py_GetRecursionLimit(void); -PyAPI_FUNC(int) Py_StackOverflowCheck(const char *where); PyAPI_FUNC(int) Py_EnterRecursiveCall(const char *where); PyAPI_FUNC(void) Py_LeaveRecursiveCall(void); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index c238cca4a2e9bf..0f700492bea2d9 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -140,7 +140,12 @@ _Py_StackOverflowCheck(PyThreadState *tstate, const char *where) static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); - return _Py_EnterRecursiveCallTstate(tstate, where); + return _Py_StackOverflowCheck(tstate, where); +} + +static inline int Py_StackOverflowCheck(const char *where) { + PyThreadState *tstate = _PyThreadState_GET(); + return _Py_StackOverflowCheck(tstate, where); } static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 5f429dc665e498..d7520905d7bf42 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1,6 +1,7 @@ /* AST Optimizer */ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_ceval.h" // Py_StackOverflowCheck #include "pycore_compile.h" // _PyASTOptimizeState #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_format.h" // F_LJUST diff --git a/Python/ceval.c b/Python/ceval.c index 51d5f88575f66c..80628a7e2a3c9d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -7464,7 +7464,8 @@ maybe_dtrace_line(_PyInterpreterFrame *frame, int Py_EnterRecursiveCall(const char *where) { - return _Py_EnterRecursiveCall(where); + PyThreadState *tstate = _PyThreadState_GET(); + return _Py_StackOverflowCheck(tstate, where); } #undef Py_LeaveRecursiveCall @@ -7473,10 +7474,3 @@ void Py_LeaveRecursiveCall(void) { _Py_LeaveRecursiveCall(); } - - -int Py_StackOverflowCheck(const char *where) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return _Py_StackOverflowCheck(tstate, where); -} diff --git a/Python/symtable.c b/Python/symtable.c index 8b7f82c76dea70..48dc6d493573d4 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_ast.h" // identifier, stmt_ty +#include "pycore_ceval.h" // Py_StackOverflowCheck #include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST() #include "pycore_parser.h" // _PyParser_ASTFromString() #include "pycore_pystate.h" // _PyThreadState_GET() From a00b59e0f513235e3e0d75840f4d7b1e562c0171 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 10:25:31 +0100 Subject: [PATCH 04/15] Increase stack size. --- Python/pystate.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/pystate.c b/Python/pystate.c index 2534a48c229442..680a62eefc18ca 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -785,9 +785,10 @@ stack_grows(void) { return -1; } } -#define DEFAULT_STACK_ALLOWANCE (1<<16) -#define SIZE_OF_RED_ZONE (1<<10) -#define SIZE_OF_YELLOW_ZONE (3<<10) +#define BLOCK_SIZE (1 << 10) +#define DEFAULT_STACK_ALLOWANCE (100 * BLOCK_SIZE) +#define SIZE_OF_RED_ZONE BLOCK_SIZE +#define SIZE_OF_YELLOW_ZONE (3 * BLOCK_SIZE) int _Py_OS_GetStackLimits(void** low, void** high); From 6c368aef59c8e4780fcfaa312b9c02bc8e01d54d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 10:31:58 +0100 Subject: [PATCH 05/15] Reduce depth required to succeed in compiler test. --- Lib/test/test_ast.py | 2 +- Lib/test/test_compile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index ed6935a04886a3..9eabc448b47c6f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -817,7 +817,7 @@ def next(self): @support.cpython_only def test_ast_recursion_limit(self): crash_depth = 100_000 - success_depth = 5_000 + success_depth = 2_000 def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 1683389db52373..1c89b3f7d4f7a1 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -547,7 +547,7 @@ def test_yet_more_evil_still_undecodable(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): crash_depth = 100_000 - success_depth = 5_000 + success_depth = 2_000 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth From 67a960e74e091b04d53f6e30cd195b80180e5185 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 11:48:46 +0100 Subject: [PATCH 06/15] Track lower bounds of stack as well. --- Include/cpython/pystate.h | 4 +-- Python/ceval.c | 37 --------------------- Python/pystate.c | 69 ++++++++++++++++++++++++++++++++------- 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 834893d1de9abc..c61b12541c475d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -101,9 +101,9 @@ struct _ts { int py_recursion_headroom; /* Allow 50 more calls to handle any errors. */ /* C stack overflow handling */ + intptr_t stack_base; intptr_t stack_limit; - intptr_t yellow_stack_limit; - intptr_t red_stack_limit; + intptr_t stack_top; int stack_in_yellow; int stack_grows; diff --git a/Python/ceval.c b/Python/ceval.c index 80628a7e2a3c9d..5f8df6befb2282 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -269,43 +269,6 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return _Py_StackOverflowCheck(tstate, where); } -int -_Py_StackOverflowCheckCall(PyThreadState *tstate, const char *where, intptr_t stack_location) -{ - assert(stack_location >= tstate->stack_limit); - assert(tstate->stack_grows); - if (stack_location >= tstate->yellow_stack_limit) { - if (_Py_UpdateStackLimits(tstate)) { - return -1; - } - } - if (stack_location < tstate->yellow_stack_limit) { - tstate->stack_limit = tstate->yellow_stack_limit; - tstate->stack_in_yellow = false; - return 0; - } - if (stack_location >= tstate->red_stack_limit) { - printf("Relative stack limits: %ld %ld %ld\n", - tstate->stack_limit - stack_location, - tstate->yellow_stack_limit - stack_location, - tstate->red_stack_limit - stack_location); - Py_FatalError("Unrecoverable C stack overflow"); - } - /* In yellow zone */ - assert(stack_location >= tstate->yellow_stack_limit && - stack_location < tstate->red_stack_limit); - if (!tstate->stack_in_yellow) { - tstate->stack_limit = INTPTR_MIN; - tstate->stack_in_yellow = true; - _PyErr_Format(tstate, PyExc_RecursionError, - "C stack overflow %s", - where); - return -1; - } - assert(tstate->stack_limit == INTPTR_MIN); - return 0; -} - /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ int diff --git a/Python/pystate.c b/Python/pystate.c index 680a62eefc18ca..1e4aff4976cb5f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -785,6 +785,13 @@ stack_grows(void) { return -1; } } + +/* We treat the top block (1k words) of the stack as the + * "red" zone. If we see a pointer in this zone, we + * abort with a Py_FatalError. The next three blocks are + * the "yellow". On first entering the yellow raise, a + * RecursionError is raised. */ + #define BLOCK_SIZE (1 << 10) #define DEFAULT_STACK_ALLOWANCE (100 * BLOCK_SIZE) #define SIZE_OF_RED_ZONE BLOCK_SIZE @@ -799,7 +806,7 @@ _Py_UpdateStackLimits(PyThreadState *tstate) #ifdef HAVE_OS_STACK_LIMITS assert(tstate->stack_grows); void *lptr, *hptr; - if (_Py_Update_StackLimits(tstate, &lptr, &hptr)) { + if (_Py_OS_GetStackLimits(&lptr, &hptr)) { return -1; } intptr_t low = (intptr_t)lptr; @@ -813,22 +820,60 @@ _Py_UpdateStackLimits(PyThreadState *tstate) low = high; high = temp; } - tstate->red_stack_limit = high - SIZE_OF_RED_ZONE; - tstate->yellow_stack_limit = tstate->red_stack_limit - SIZE_OF_YELLOW_ZONE; + tstate->stack_base = low; + tstate->stack_top = high; #else char addr; intptr_t here = ((uintptr_t)&addr)/SIZEOF_VOID_P; here *= tstate->stack_grows; - if (here < tstate->red_stack_limit - DEFAULT_STACK_ALLOWANCE || - here > tstate->red_stack_limit + DEFAULT_STACK_ALLOWANCE/2) + if (here < tstate->stack_base || + here > tstate->stack_top + 8*BLOCK_SIZE) { - /* Either uninitialized or - * sufficiently out of bounds, that we must have the wrong thread. */ - intptr_t high = here + DEFAULT_STACK_ALLOWANCE; - tstate->red_stack_limit = high - SIZE_OF_RED_ZONE; - tstate->yellow_stack_limit = tstate->red_stack_limit - SIZE_OF_YELLOW_ZONE; + /* Either uninitialized or, + * sufficiently out of bounds that we must have the wrong thread. */ + tstate->stack_base = here - BLOCK_SIZE; + tstate->stack_top = tstate->stack_base + DEFAULT_STACK_ALLOWANCE; } #endif + if (!tstate->stack_in_yellow) { + tstate->stack_limit = tstate->stack_top - SIZE_OF_RED_ZONE - SIZE_OF_YELLOW_ZONE; + } + return 0; +} + +int +_Py_StackOverflowCheckCall(PyThreadState *tstate, const char *where, intptr_t stack_location) +{ + assert(stack_location >= tstate->stack_limit); + assert(tstate->stack_grows); + if (_Py_UpdateStackLimits(tstate)) { + return -1; + } + intptr_t yellow_start = tstate->stack_top - SIZE_OF_RED_ZONE - SIZE_OF_YELLOW_ZONE; + if (stack_location < yellow_start) { + tstate->stack_limit = yellow_start; + tstate->stack_in_yellow = 0; + return 0; + } + intptr_t red_start = tstate->stack_top - SIZE_OF_RED_ZONE; + if (stack_location >= red_start) { + printf("Relative stack values. base: %ld limit: %ld top: %ld\n", + tstate->stack_base - stack_location, + tstate->stack_limit - stack_location, + tstate->stack_top - stack_location); + Py_FatalError("Unrecoverable C stack overflow"); + } + /* In yellow zone */ + assert(stack_location >= yellow_start && stack_location < red_start); + if (!tstate->stack_in_yellow) { + tstate->stack_limit = INTPTR_MIN; + tstate->stack_in_yellow = 1; + _PyErr_Format(tstate, PyExc_RecursionError, + "C stack overflow %s", + where); + return -1; + } + assert(tstate->stack_limit == INTPTR_MIN); return 0; } @@ -874,8 +919,8 @@ init_threadstate(PyThreadState *tstate, tstate->stack_grows = stack_grows(); tstate->stack_limit = INTPTR_MIN; - tstate->yellow_stack_limit = INTPTR_MIN; - tstate->red_stack_limit = INTPTR_MIN; + tstate->stack_base = INTPTR_MAX; + tstate->stack_top = INTPTR_MIN; tstate->stack_in_yellow = 0; tstate->_initialized = 1; From 42b7e70f4d713c6862743a92b7b3379e1f93440a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 12:41:36 +0100 Subject: [PATCH 07/15] _Py_StackOverflowCheckCall is not part of the API. --- Include/internal/pycore_ceval.h | 2 +- Python/pystate.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 0f700492bea2d9..c5f830006d3dab 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -119,7 +119,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCall( const char *where); -PyAPI_FUNC(int) _Py_StackOverflowCheckCall( +int _Py_StackOverflowCheckCall( PyThreadState *tstate, const char *where, intptr_t scaled_location); diff --git a/Python/pystate.c b/Python/pystate.c index 1e4aff4976cb5f..a59543138513ce 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -830,7 +830,7 @@ _Py_UpdateStackLimits(PyThreadState *tstate) here > tstate->stack_top + 8*BLOCK_SIZE) { /* Either uninitialized or, - * sufficiently out of bounds that we must have the wrong thread. */ + * sufficiently out of bounds that we must be in a new thread. */ tstate->stack_base = here - BLOCK_SIZE; tstate->stack_top = tstate->stack_base + DEFAULT_STACK_ALLOWANCE; } From e57cbd29bc0990d1c8c5b401886ba2b070dda175 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 12:42:03 +0100 Subject: [PATCH 08/15] Further reduce compiler test depths. --- Lib/test/test_ast.py | 2 +- Lib/test/test_compile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 9eabc448b47c6f..71ff7c9edb56f3 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -817,7 +817,7 @@ def next(self): @support.cpython_only def test_ast_recursion_limit(self): crash_depth = 100_000 - success_depth = 2_000 + success_depth = 1500 def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 1c89b3f7d4f7a1..dc35ea0787a4cb 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -547,7 +547,7 @@ def test_yet_more_evil_still_undecodable(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): crash_depth = 100_000 - success_depth = 2_000 + success_depth = 1500 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth From ee2511cda2de67126241444a50ee388a307d61ee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 12:53:33 +0100 Subject: [PATCH 09/15] Add required symbol back. --- Include/internal/pycore_ceval.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index c5f830006d3dab..0f700492bea2d9 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -119,7 +119,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCall( const char *where); -int _Py_StackOverflowCheckCall( +PyAPI_FUNC(int) _Py_StackOverflowCheckCall( PyThreadState *tstate, const char *where, intptr_t scaled_location); From 8ea7c66286e6c85431e5bf0fe44de92f0457e769 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 13:29:36 +0100 Subject: [PATCH 10/15] Futher reduce depths to allow ASAN and Windows debug build tests to pass. --- Lib/test/test_ast.py | 2 +- Lib/test/test_compile.py | 2 +- Lib/test/test_sys_settrace.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 71ff7c9edb56f3..435c8e748306bb 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -817,7 +817,7 @@ def next(self): @support.cpython_only def test_ast_recursion_limit(self): crash_depth = 100_000 - success_depth = 1500 + success_depth = 1000 def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index dc35ea0787a4cb..4f63c0d27afc21 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -547,7 +547,7 @@ def test_yet_more_evil_still_undecodable(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): crash_depth = 100_000 - success_depth = 1500 + success_depth = 1000 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 9f1aa81dbcd249..d4e1bfdb74d7ae 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2751,16 +2751,17 @@ def test_trace_unpack_long_sequence(self): self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) def test_trace_lots_of_globals(self): + COUNT = 700 code = """if 1: def f(): return ( {} ) - """.format("\n+\n".join(f"var{i}\n" for i in range(1000))) - ns = {f"var{i}": i for i in range(1000)} + """.format("\n+\n".join(f"var{i}\n" for i in range(700))) + ns = {f"var{i}": i for i in range(700)} exec(code, ns) counts = self.count_traces(ns["f"]) - self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1}) + self.assertEqual(counts, {'call': 1, 'line': 1400, 'return': 1}) class TestEdgeCases(unittest.TestCase): From 4afacdb134e2d7fb6a43c2ea5777448ceadccfd9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 13:46:14 +0100 Subject: [PATCH 11/15] Futher reduce depths to allow ASAN build tests to pass. --- Lib/test/test_compile.py | 3 +-- Lib/test/test_dynamic.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4f63c0d27afc21..cbafe194b81b93 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -111,8 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - # default: 1000 * 2.5 = 2500 repetitions - repeat = int(sys.getrecursionlimit() * 2.5) + repeat = 1000 longexpr = 'x = x or ' + '-x' * repeat g = {} code = ''' diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index e896fc703c9a8b..62241dd10f6c9f 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -140,9 +140,7 @@ class MyGlobals(dict): def __missing__(self, key): return int(key.removeprefix("_number_")) - # 1,000 on most systems - limit = sys.getrecursionlimit() - code = "lambda: " + "+".join(f"_number_{i}" for i in range(limit)) + code = "lambda: " + "+".join(f"_number_{i}" for i in range(700)) sum_func = eval(code, MyGlobals()) expected = sum(range(limit)) # Warm up the the function for quickening (PEP 659) From a6ae7d30ce284f1c8cddc33d37622ce374e6e21f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 14:08:06 +0100 Subject: [PATCH 12/15] Futher reduce depths to allow ASAN build tests to pass. --- Lib/test/test_compile.py | 2 +- Lib/test/test_dynamic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index cbafe194b81b93..dabbdbd38fecb4 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -111,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 1000 + repeat = 700 longexpr = 'x = x or ' + '-x' * repeat g = {} code = ''' diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 62241dd10f6c9f..ede05567ee81ac 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -142,7 +142,7 @@ def __missing__(self, key): code = "lambda: " + "+".join(f"_number_{i}" for i in range(700)) sum_func = eval(code, MyGlobals()) - expected = sum(range(limit)) + expected = sum(range(700)) # Warm up the the function for quickening (PEP 659) for _ in range(30): self.assertEqual(sum_func(), expected) From a7fad4d988b37d236e6871cb0125db358351df78 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 14:20:08 +0100 Subject: [PATCH 13/15] Add tests for super deep recursion. --- Lib/test/test_call.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index c00de27b265d27..e1773e71c60cd1 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -813,6 +813,44 @@ def test_multiple_values(self): with self.check_raises_type_error(msg): A().method_two_args("x", "y", x="oops") +@cpython_only +class TestRecursion(unittest.TestCase): + + def test_super_deep(self): + + def recurse(n): + if n: + recurse(n-1) + + def py_recurse(n, m): + if n: + py_recurse(n-1, m) + else: + c_py_recurse(m-1) + + def c_recurse(n): + if n: + _testcapi.pyobject_fastcall(c_recurse, (n-1,)) + + def c_py_recurse(m): + if m: + _testcapi.pyobject_fastcall(py_recurse, (1000, m)) + + depth = sys.getrecursionlimit() + sys.setrecursionlimit(100_000) + try: + recurse(90_000) + with self.assertRaises(RecursionError): + recurse(101_000) + c_recurse(100) + with self.assertRaises(RecursionError): + c_recurse(90_000) + c_py_recurse(90) + with self.assertRaises(RecursionError): + c_py_recurse(100_000) + finally: + sys.setrecursionlimit(depth) + if __name__ == "__main__": unittest.main() From f915e01fee342581e906b00fc691e47fccb41813 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 14:34:57 +0100 Subject: [PATCH 14/15] Futher reduce depths to allow ASAN build tests to pass. --- Lib/test/test_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index dabbdbd38fecb4..2134f645964467 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -546,7 +546,7 @@ def test_yet_more_evil_still_undecodable(self): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): crash_depth = 100_000 - success_depth = 1000 + success_depth = 700 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth From 6976e5926724805705ca2546872f156d7413dab9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 14:46:17 +0100 Subject: [PATCH 15/15] Skip test for sanitizer. --- Lib/test/test_compile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 2134f645964467..5f164d9c997632 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -544,9 +544,11 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_if_sanitizer(memory=True, address=True, + reason= "sanitizer consumes too much stack space") def test_compiler_recursion_limit(self): crash_depth = 100_000 - success_depth = 700 + success_depth = 1000 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth