gh-134761: Use deferred reference counting for threading concurrency primitives

Lib/test/test_sys.py

@@ -1343,6 +1343,25 @@ def test_pystats(self):
  def test_disable_gil_abi(self):
  self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED)
 
+ @test.support.cpython_only
+ @unittest.skipUnless(hasattr(sys, '_defer_refcount'), "requires _defer_refcount()")
+ def test_defer_refcount(self):
+ _testinternalcapi = import_helper.import_module('_testinternalcapi')
+
+ class Test:
+ pass
+
+ ref = Test()
+ if support.Py_GIL_DISABLED:
+ self.assertTrue(sys._defer_refcount(ref))
+ self.assertTrue(_testinternalcapi.has_deferred_refcount(ref))
+ self.assertFalse(sys._defer_refcount(ref))
+ self.assertFalse(sys._defer_refcount(42))
+ else:
+ self.assertFalse(sys._defer_refcount(ref))
+ self.assertFalse(_testinternalcapi.has_deferred_refcount(ref))
+ self.assertFalse(sys._defer_refcount(42))
+
 
 @test.support.cpython_only
 class UnraisableHookTest(unittest.TestCase):

Lib/threading.py

@@ -73,6 +73,11 @@
 _profile_hook = None
 _trace_hook = None
 
+def _defer_refcount(op):
+ """Improve multithreaded scaling on the free-threade build."""
+ if hasattr(_sys, "_defer_refcount"):
+ _sys._defer_refcount(op)
+
 def setprofile(func):
  """Set a profile function for all threads started from the threading module.
 
@@ -298,6 +303,7 @@ def __init__(self, lock=None):
  if hasattr(lock, '_is_owned'):
  self._is_owned = lock._is_owned
  self._waiters = _deque()
+ _defer_refcount(self)
 
  def _at_fork_reinit(self):
  self._lock._at_fork_reinit()
@@ -466,6 +472,7 @@ def __init__(self, value=1):
  raise ValueError("semaphore initial value must be >= 0")
  self._cond = Condition(Lock())
  self._value = value
+ _defer_refcount(self)
 
  def __repr__(self):
  cls = self.__class__
@@ -595,6 +602,7 @@ class Event:
  def __init__(self):
  self._cond = Condition(Lock())
  self._flag = False
+ _defer_refcount(self)
 
  def __repr__(self):
  cls = self.__class__
@@ -700,6 +708,7 @@ def __init__(self, parties, action=None, timeout=None):
  self._parties = parties
  self._state = 0 # 0 filling, 1 draining, -1 resetting, -2 broken
  self._count = 0
+ _defer_refcount(self)
 
  def __repr__(self):
  cls = self.__class__

Misc/NEWS.d/next/Library/2025-05-26-19-49-16.gh-issue-134761.LeMefI.rst

@@ -0,0 +1,2 @@
+Improve performance when using :mod:`threading` primitives across multiple
+threads.

Modules/_threadmodule.c

@@ -951,6 +951,7 @@ lock_new_impl(PyTypeObject *type)
  return NULL;
  }
  self->lock = (PyMutex){0};
+ _PyObject_SetDeferredRefcount((PyObject *)self);
  return (PyObject *)self;
 }
 
@@ -1222,6 +1223,7 @@ rlock_new_impl(PyTypeObject *type)
  return NULL;
  }
  self->lock = (_PyRecursiveMutex){0};
+ _PyObject_SetDeferredRefcount((PyObject *)self);
  return (PyObject *) self;
 }
 

Python/sysmodule.c

@@ -15,6 +15,7 @@ Data members:
 */
 
 #include "Python.h"
+#include "object.h"
 #include "pycore_audit.h" // _Py_AuditHookEntry
 #include "pycore_call.h" // _PyObject_CallNoArgs()
 #include "pycore_ceval.h" // _PyEval_SetAsyncGenFinalizer()
@@ -2653,6 +2654,23 @@ sys__is_gil_enabled_impl(PyObject *module)
 #endif
 }
 
+/*[clinic input]
+sys._defer_refcount -> bool
+
+ op: object
+ /
+
+Defer reference counting for the object, allowing for better scaling across multiple threads.
+
+This function should be used for specialized purposes only.
+[clinic start generated code]*/
+
+static int
+sys__defer_refcount_impl(PyObject *module, PyObject *op)
+/*[clinic end generated code: output=3b965122056085f5 input=a081971a76c49e64]*/
+{
+ return PyUnstable_Object_EnableDeferredRefcount(op);
+}
 
 #ifndef MS_WINDOWS
 static PerfMapState perf_map_state;
@@ -2834,6 +2852,7 @@ static PyMethodDef sys_methods[] ={
  SYS__GET_CPU_COUNT_CONFIG_METHODDEF
  SYS__IS_GIL_ENABLED_METHODDEF
  SYS__DUMP_TRACELETS_METHODDEF
+ SYS__DEFER_REFCOUNT_METHODDEF
 {NULL, NULL} // sentinel
 };