pythongh-123619: Add an unstable C API function for enabling deferred reference counting (pythonGH-123635)

Co-authored-by: Sam Gross <[email protected]>

Author: ZeroIntensity

Doc/c-api/object.rst

@@ -575,3 +575,27 @@ Object Protocol
    has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set.
 
    .. versionadded:: 3.13
+
+.. c:function:: int PyUnstable_Object_EnableDeferredRefcount(PyObject *obj)
+
+   Enable `deferred reference counting <https://peps.python.org/pep-0703/#deferred-reference-counting>`_ on *obj*,
+   if supported by the runtime.  In the :term:`free-threaded <free threading>` build,
+   this allows the interpreter to avoid reference count adjustments to *obj*,
+   which may improve multi-threaded performance.  The tradeoff is
+   that *obj* will only be deallocated by the tracing garbage collector.
+
+   This function returns ``1`` if deferred reference counting is enabled on *obj*
+   (including when it was enabled before the call),
+   and ``0`` if deferred reference counting is not supported or if the hint was
+   ignored by the runtime. This function is thread-safe, and cannot fail.
+
+   This function does nothing on builds with the :term:`GIL` enabled, which do
+   not support deferred reference counting. This also does nothing if *obj* is not
+   an object tracked by the garbage collector (see :func:`gc.is_tracked` and
+   :c:func:`PyObject_GC_IsTracked`).
+
+   This function is intended to be used soon after *obj* is created,
+   by the code that creates it.
+
+   .. versionadded:: next
+

Doc/whatsnew/3.14.rst

@@ -890,6 +890,9 @@ New features
 * Add :c:func:`PyType_Freeze` function to make a type immutable.
   (Contributed by Victor Stinner in :gh:`121654`.)
 
+* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling
+  deferred reference counting, as outlined in :pep:`703`.
+
 Porting to Python 3.14
 ----------------------
 

Include/cpython/object.h

@@ -527,3 +527,10 @@ typedef enum {
 typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *);
 PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data);
 PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**);
+
+/* Enable PEP-703 deferred reference counting on the object.
+ *
+ * Returns 1 if deferred reference counting was successfully enabled, and
+ * 0 if the runtime ignored it. This function cannot fail.
+ */
+PyAPI_FUNC(int) PyUnstable_Object_EnableDeferredRefcount(PyObject *);

Lib/test/test_capi/test_object.py

@@ -1,10 +1,13 @@
 import enum
 import unittest
+from test import support
 from test.support import import_helper
 from test.support import os_helper
+from test.support import threading_helper
 
 _testlimitedcapi = import_helper.import_module('_testlimitedcapi')
 _testcapi = import_helper.import_module('_testcapi')
+_testinternalcapi = import_helper.import_module('_testinternalcapi')
 
 
 class Constant(enum.IntEnum):
@@ -131,5 +134,48 @@ def test_ClearWeakRefsNoCallbacks_no_weakref_support(self):
         _testcapi.pyobject_clear_weakrefs_no_callbacks(obj)
 
 
+class EnableDeferredRefcountingTest(unittest.TestCase):
+    """Test PyUnstable_Object_EnableDeferredRefcount"""
+    @support.requires_resource("cpu")
+    def test_enable_deferred_refcount(self):
+        from threading import Thread
+
+        self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0)
+        foo = []
+        self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED))
+
+        # Make sure reference counting works on foo now
+        self.assertEqual(foo, [])
+        if support.Py_GIL_DISABLED:
+            self.assertTrue(_testinternalcapi.has_deferred_refcount(foo))
+
+        # Make sure that PyUnstable_Object_EnableDeferredRefcount is thread safe
+        def silly_func(obj):
+            self.assertIn(
+                _testcapi.pyobject_enable_deferred_refcount(obj),
+                (0, 1)
+            )
+
+        silly_list = [1, 2, 3]
+        threads = [
+            Thread(target=silly_func, args=(silly_list,)) for _ in range(5)
+        ]
+
+        with threading_helper.catch_threading_exception() as cm:
+            for t in threads:
+                t.start()
+
+            for i in range(10):
+                silly_list.append(i)
+
+            for t in threads:
+                t.join()
+
+            self.assertIsNone(cm.exc_value)
+
+        if support.Py_GIL_DISABLED:
+            self.assertTrue(_testinternalcapi.has_deferred_refcount(silly_list))
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/C_API/2024-09-03-13-33-33.gh-issue-123619.HhgUUI.rst

@@ -0,0 +1,2 @@
+Added the :c:func:`PyUnstable_Object_EnableDeferredRefcount` function for
+enabling :pep:`703` deferred reference counting.

Modules/_testcapi/object.c

@@ -124,13 +124,20 @@ pyobject_clear_weakrefs_no_callbacks(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+pyobject_enable_deferred_refcount(PyObject *self, PyObject *obj)
+{
+    int result = PyUnstable_Object_EnableDeferredRefcount(obj);
+    return PyLong_FromLong(result);
+}
+
 static PyMethodDef test_methods[] = {
     {"call_pyobject_print", call_pyobject_print, METH_VARARGS},
     {"pyobject_print_null", pyobject_print_null, METH_VARARGS},
     {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS},
     {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS},
     {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O},
-
+    {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O},
     {NULL},
 };
 

Modules/_testinternalcapi.c

@@ -2069,6 +2069,14 @@ identify_type_slot_wrappers(PyObject *self, PyObject *Py_UNUSED(ignored))
     return _PyType_GetSlotWrapperNames();
 }
 
+
+static PyObject *
+has_deferred_refcount(PyObject *self, PyObject *op)
+{
+    return PyBool_FromLong(_PyObject_HasDeferredRefcount(op));
+}
+
+
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -2165,6 +2173,7 @@ static PyMethodDef module_functions[] = {
     GH_119213_GETARGS_METHODDEF
     {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS},
     {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS},
+    {"has_deferred_refcount", has_deferred_refcount, METH_O},
     {NULL, NULL} /* sentinel */
 };
 

Objects/object.c

@@ -2519,6 +2519,35 @@ _PyObject_SetDeferredRefcount(PyObject *op)
 #endif
 }
 
+int
+PyUnstable_Object_EnableDeferredRefcount(PyObject *op)
+{
+#ifdef Py_GIL_DISABLED
+    if (!PyType_IS_GC(Py_TYPE(op))) {
+        // Deferred reference counting doesn't work
+        // on untracked types.
+        return 0;
+    }
+
+    uint8_t bits = _Py_atomic_load_uint8(&op->ob_gc_bits);
+    if ((bits & _PyGC_BITS_DEFERRED) != 0)
+    {
+        // Nothing to do.
+        return 0;
+    }
+
+    if (_Py_atomic_compare_exchange_uint8(&op->ob_gc_bits, &bits, bits | _PyGC_BITS_DEFERRED) == 0)
+    {
+        // Someone beat us to it!
+        return 0;
+    }
+    _Py_atomic_add_ssize(&op->ob_ref_shared, _Py_REF_SHARED(_Py_REF_DEFERRED, 0));
+    return 1;
+#else
+    return 0;
+#endif
+}
+
 void
 _Py_ResurrectReference(PyObject *op)
 {