Port basics of stack spilling

Author: markshannon

Doc/whatsnew/3.13.rst

@@ -753,9 +753,14 @@ fractions
 gc
 --
 
-* The cyclic garbage collector is now incremental, which changes the meanings
-  of the results of :meth:`gc.get_threshold` and :meth:`gc.set_threshold` as
-  well as :meth:`gc.get_count` and :meth:`gc.get_stats`.
+* The cyclic garbage collector is now incremental. Incremental garbage collection
+  scans only a portion of the heap at each collection, so can only collect a
+  portion of the potential cycles. This means that pauses are generally a lot shorter,
+  although memory use might increase as there can be a longer delay before cycles are
+  collected. All cyclic garbage will be collected eventually.
+
+  The meanings of :meth:`gc.get_threshold` and :meth:`gc.set_threshold` as
+  well as :meth:`gc.get_count` and :meth:`gc.get_stats` have changed:
 
   * :meth:`gc.get_threshold` returns a three-item tuple for backwards compatibility.
     The first value is the threshold for young collections, as before; the second

Include/cpython/pystate.h

@@ -192,6 +192,7 @@ struct _ts {
     PyObject *previous_executor;
 
     uint64_t dict_global_version;
+    int sp_cached; /* Only used in debug builds */
 };
 
 #ifdef Py_DEBUG

Include/internal/pycore_frame.h

@@ -64,7 +64,7 @@ typedef struct _PyInterpreterFrame {
     PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
     PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
     _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */
-    int stacktop;  /* Offset of TOS from localsplus  */
+    _PyStackRef *stackpointer;
     uint16_t return_offset;  /* Only relevant during a function call */
     char owner;
     /* Locals and stack */
@@ -84,20 +84,20 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) {
 }
 
 static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) {
-    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
-    assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1]));
-    return f->localsplus[f->stacktop-1];
+    assert(f->stackpointer >  f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
+    assert(!PyStackRef_IsNull(f->stackpointer[-1]));
+    return f->stackpointer[-1];
 }
 
 static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) {
-    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
-    f->stacktop--;
-    return f->localsplus[f->stacktop];
+    assert(f->stackpointer >  f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
+    f->stackpointer--;
+    return *f->stackpointer;
 }
 
 static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) {
-    f->localsplus[f->stacktop] = value;
-    f->stacktop++;
+    *f->stackpointer = value;
+    f->stackpointer++;
 }
 
 #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *)))
@@ -113,9 +113,12 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code)
 
 static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
 {
-    assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus);
     *dest = *src;
-    for (int i = 1; i < src->stacktop; i++) {
+    assert(src->stackpointer != NULL);
+    int stacktop = (int)(src->stackpointer - src->localsplus);
+    assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus);
+    dest->stackpointer = dest->localsplus + stacktop;
+    for (int i = 1; i < stacktop; i++) {
         dest->localsplus[i] = src->localsplus[i];
     }
     // Don't leave a dangling pointer to the old frame when creating generators
@@ -137,7 +140,7 @@ _PyFrame_Initialize(
     frame->f_builtins = func->func_builtins;
     frame->f_globals = func->func_globals;
     frame->f_locals = locals;
-    frame->stacktop = code->co_nlocalsplus;
+    frame->stackpointer = frame->localsplus + code->co_nlocalsplus;
     frame->frame_obj = NULL;
     frame->instr_ptr = _PyCode_CODE(code);
     frame->return_offset = 0;
@@ -157,22 +160,29 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
     return frame->localsplus;
 }
 
-/* Fetches the stack pointer, and sets stacktop to -1.
-   Having stacktop <= 0 ensures that invalid
-   values are not visible to the cycle GC.
-   We choose -1 rather than 0 to assist debugging. */
+/* Fetches the stack pointer, and sets stackpointer to NULL.
+   Having stackpointer == NULL ensures that invalid
+   values are not visible to the cycle GC. */
 static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
-    _PyStackRef *sp = frame->localsplus + frame->stacktop;
-    frame->stacktop = -1;
+#ifdef Py_DEBUG
+    PyThreadState_GET()->sp_cached++;
+#endif
+    assert(frame->stackpointer != NULL);
+    _PyStackRef *sp = frame->stackpointer;
+    frame->stackpointer = NULL;
     return sp;
 }
 
 static inline void
 _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 {
-    frame->stacktop = (int)(stack_pointer - frame->localsplus);
+#ifdef Py_DEBUG
+    PyThreadState_GET()->sp_cached--;
+#endif
+    assert(frame->stackpointer == NULL);
+    frame->stackpointer = stack_pointer;
 }
 
 /* Determine whether a frame is incomplete.
@@ -300,7 +310,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->f_globals = NULL;
 #endif
     frame->f_locals = NULL;
-    frame->stacktop = code->co_nlocalsplus + stackdepth;
+    assert(stackdepth <= code->co_stacksize);
+    frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth;
     frame->frame_obj = NULL;
     frame->instr_ptr = _PyCode_CODE(code);
     frame->owner = FRAME_OWNED_BY_THREAD;

Lib/test/test_sys.py

@@ -1603,7 +1603,7 @@ class C(object): pass
         def func():
             return sys._getframe()
         x = func()
-        check(x, size('3Pi2cP7P2ic??2P'))
+        check(x, size('3Pi2cP9Phc2P'))
         # function
         def func(): pass
         check(func, size('16Pi'))
@@ -1620,7 +1620,7 @@ def bar(cls):
             check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size('PP4P4c7P2ic??2P'))
+        check(get_gen(), size('PP4P4cP9PhcP'))
         # iterator
         check(iter('abc'), size('lP'))
         # callable-iterator

Objects/frameobject.c

@@ -1620,8 +1620,10 @@ frame_dealloc(PyFrameObject *f)
         Py_CLEAR(frame->f_funcobj);
         Py_CLEAR(frame->f_locals);
         _PyStackRef *locals = _PyFrame_GetLocalsArray(frame);
-        for (int i = 0; i < frame->stacktop; i++) {
-            PyStackRef_CLEAR(locals[i]);
+        _PyStackRef *sp = frame->stackpointer;
+        while (sp > locals) {
+            sp--;
+            PyStackRef_CLEAR(*sp);
         }
     }
     Py_CLEAR(f->f_back);
@@ -1653,11 +1655,13 @@ frame_tp_clear(PyFrameObject *f)
 
     /* locals and stack */
     _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame);
-    assert(f->f_frame->stacktop >= 0);
-    for (int i = 0; i < f->f_frame->stacktop; i++) {
-        PyStackRef_CLEAR(locals[i]);
+    _PyStackRef *sp = f->f_frame->stackpointer;
+    assert(sp >= locals);
+    while (sp > locals) {
+            sp--;
+            PyStackRef_CLEAR(*sp);
     }
-    f->f_frame->stacktop = 0;
+    f->f_frame->stackpointer = locals;
     Py_CLEAR(f->f_frame->f_locals);
     return 0;
 }
@@ -1875,7 +1879,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i,
     }
 
     PyObject *value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]);
-    if (frame->stacktop) {
+    if (frame->stackpointer > frame->localsplus) {
         if (kind & CO_FAST_FREE) {
             // The cell was set by COPY_FREE_VARS.
             assert(value != NULL && PyCell_Check(value));

Objects/object.c

@@ -2842,6 +2842,9 @@ _Py_Dealloc(PyObject *op)
     destructor dealloc = type->tp_dealloc;
 #ifdef Py_DEBUG
     PyThreadState *tstate = _PyThreadState_GET();
+#ifndef Py_GIL_DISABLED
+    assert(tstate->sp_cached == 0);
+#endif
     PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL;
     // Keep the old exception type alive to prevent undefined behavior
     // on (tstate->curexc_type != old_exc_type) below

Python/bytecodes.c

@@ -863,7 +863,9 @@ dummy_func(
 
         inst(CALL_INTRINSIC_1, (value -- res)) {
             assert(oparg <= MAX_INTRINSIC_1);
-            PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value));
+            ESCAPING_CALL(
+                PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value))
+            );
             DECREF_INPUTS();
             ERROR_IF(res_o == NULL, error);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -873,8 +875,9 @@ dummy_func(
             assert(oparg <= MAX_INTRINSIC_2);
             PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st);
             PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st);
-
-            PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
+            ESCAPING_CALL(
+                PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1)
+            );
             DECREF_INPUTS();
             ERROR_IF(res_o == NULL, error);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -2406,7 +2409,9 @@ dummy_func(
             STAT_INC(STORE_ATTR, hit);
             PyObject *old_value = *(PyObject **)addr;
             *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
-            Py_XDECREF(old_value);
+            if (old_value != NULL) {
+                Py_DECREF(old_value);
+            }
             PyStackRef_CLOSE(owner);
         }
 

Python/ceval.c

@@ -52,36 +52,50 @@
 #  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
 #endif
 
-#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED)
-// GH-89279: The MSVC compiler does not inline these static inline functions
-// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
-// the limit of PGO, and that limit cannot be configured.
-// Define them as macros to make sure that they are always inlined by the
-// preprocessor.
-// TODO: implement Py_DECREF macro for Py_GIL_DISABLED
-
-#undef Py_DECREF
-#define Py_DECREF(arg) \
+#ifdef Py_GIL_DISABLED
+
+#define INTERPRETER_REFCLOSE PyStackRef_CLOSE
+
+#else
+
+#ifdef Py_REF_DEBUG
+static inline void interpreter_refclose(_PyInterpreterFrame *frame, _PyStackRef *sp, const char *filename, int lineno, _PyStackRef ref)
+{
+    PyObject *op = PyStackRef_AsPyObjectBorrow(ref);
+    if (op->ob_refcnt <= 0) {
+        _Py_NegativeRefcount(filename, lineno, op);
+    }
+    if (_Py_IsImmortal(op)) {
+        return;
+    }
+    _Py_DECREF_STAT_INC();
+    _Py_DECREF_DecRefTotal();
+    if (--op->ob_refcnt == 0) {
+        _PyFrame_SetStackPointer(frame, sp);
+        _Py_Dealloc(op);
+        _PyFrame_GetStackPointer(frame);
+    }
+}
+#define INTERPRETER_REFCLOSE(ref) interpreter_refclose(frame, stack_pointer, __FILE__, __LINE__, ref)
+
+#else
+
+#define INTERPRETER_REFCLOSE(ref) \
     do { \
-        PyObject *op = _PyObject_CAST(arg); \
+        PyObject *op = PyStackRef_AsPyObjectBorrow(ref); \
         if (_Py_IsImmortal(op)) { \
             break; \
         } \
         _Py_DECREF_STAT_INC(); \
         if (--op->ob_refcnt == 0) { \
-            destructor dealloc = Py_TYPE(op)->tp_dealloc; \
-            (*dealloc)(op); \
+            _PyFrame_SetStackPointer(frame, stack_pointer); \
+            _Py_Dealloc(op); \
+            stack_pointer = _PyFrame_GetStackPointer(frame); \
         } \
     } while (0)
 
-#undef Py_XDECREF
-#define Py_XDECREF(arg) \
-    do { \
-        PyObject *xop = _PyObject_CAST(arg); \
-        if (xop != NULL) { \
-            Py_DECREF(xop); \
-        } \
-    } while (0)
+#endif // Py_REF_DEBUG
+#endif // Py_GIL_DISABLED
 
 #undef Py_IS_TYPE
 #define Py_IS_TYPE(ob, type) \
@@ -100,7 +114,6 @@
             d(op); \
         } \
     } while (0)
-#endif
 
 
 #ifdef LLTRACE
@@ -773,7 +786,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #endif
     entry_frame.f_executable = Py_None;
     entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1;
-    entry_frame.stacktop = 0;
+    entry_frame.stackpointer = entry_frame.localsplus;
     entry_frame.owner = FRAME_OWNED_BY_CSTACK;
     entry_frame.return_offset = 0;
     /* Push frame */
@@ -936,7 +949,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
                 PyTraceBack_Here(f);
             }
         }
-        monitor_raise(tstate, frame, next_instr-1);
+        ESCAPING_CALL(monitor_raise(tstate, frame, next_instr-1));
 exception_unwind:
         {
             /* We can't use frame->instr_ptr here, as RERAISE may have set it */
@@ -949,7 +962,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
                 /* Pop remaining stack entries. */
                 _PyStackRef *stackbase = _PyFrame_Stackbase(frame);
                 while (stack_pointer > stackbase) {
-                    PyStackRef_XCLOSE(POP());
+                    _PyStackRef ref = POP();
+                    if (!PyStackRef_IsNull(ref)) {
+                        INTERPRETER_REFCLOSE(ref);
+                    };
                 }
                 assert(STACK_LEVEL() == 0);
                 _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -960,7 +976,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             assert(STACK_LEVEL() >= level);
             _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level;
             while (stack_pointer > new_top) {
-                PyStackRef_XCLOSE(POP());
+                _PyStackRef ref = POP();
+                if (!PyStackRef_IsNull(ref)) {
+                    INTERPRETER_REFCLOSE(ref);
+                };
             }
             if (lasti) {
                 int frame_lasti = _PyInterpreterFrame_LASTI(frame);
@@ -979,7 +998,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
             PUSH(PyStackRef_FromPyObjectSteal(exc));
             next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler;
 
-            if (monitor_handled(tstate, frame, next_instr, exc) < 0) {
+            ESCAPING_CALL(int err = monitor_handled(tstate, frame, next_instr, exc));
+            if (err < 0) {
                 goto exception_unwind;
             }
             /* Resume normal execution */