gh-115999: Specialize LOAD_GLOBAL in free-threaded builds

Include/internal/pycore_dict.h

@@ -90,6 +90,17 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *);
 extern uint32_t _PyDictKeys_GetVersionForCurrentState(
  PyInterpreterState *interp, PyDictKeysObject *dictkeys);
 
+/* Gets a version number unique to the current state of the keys of dict, if possible.
+ *
+ * In free-threaded builds ensures that the dict can be used for lock-free
+ * reads if a version was assigned.
+ *
+ * The caller must hold the per-object lock on dict.
+ *
+ * Returns the version number, or zero if it was not possible to get a version number. */
+extern uint32_t _PyDict_GetKeysVersionForCurrentState(
+ PyInterpreterState *interp, PyDictObject *dict);
+
 extern size_t _PyDict_KeysSize(PyDictKeysObject *keys);
 
 extern void _PyDictKeys_DecRef(PyDictKeysObject *keys);

Include/internal/pycore_object.h

@@ -14,6 +14,7 @@ extern "C"{
 #include "pycore_interp.h" // PyInterpreterState.gc
 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
+#include "pycore_stackref.h"
 #include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow()
 
 // This value is added to `ob_ref_shared` for objects that use deferred
@@ -595,6 +596,20 @@ _Py_TryIncrefCompare(PyObject **src, PyObject *op)
  return 1;
 }
 
+static inline int
+_Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out)
+{
+ if (_Py_IsImmortal(op) || _PyObject_HasDeferredRefcount(op)){
+ *out = (_PyStackRef){.bits = (intptr_t)op | Py_TAG_DEFERRED };
+ return 1;
+ }
+ if (_Py_TryIncrefCompare(src, op)){
+ *out = PyStackRef_FromPyObjectSteal(op);
+ return 1;
+ }
+ return 0;
+}
+
 /* Loads and increfs an object from ptr, which may contain a NULL value.
  Safe with concurrent (atomic) updates to ptr.
  NOTE: The writer must set maybe-weakref on the stored object! */

Lib/test/test_opcache.py

@@ -546,7 +546,6 @@ def count_args(self, *args):
 
 
 @threading_helper.requires_working_threading()
-@requires_specialization
 class TestRacesDoNotCrash(TestBase):
  # Careful with these. Bigger numbers have a higher chance of catching bugs,
  # but you can also burn through a *ton* of type/dict/function versions:
@@ -588,6 +587,7 @@ def assert_races_do_not_crash(
  for writer in writers:
  writer.join()
 
+ @requires_specialization
  def test_binary_subscr_getitem(self):
  def get_items():
  class C:
@@ -617,6 +617,7 @@ def write(items):
  opname = "BINARY_SUBSCR_GETITEM"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_binary_subscr_list_int(self):
  def get_items():
  items = []
@@ -640,6 +641,7 @@ def write(items):
  opname = "BINARY_SUBSCR_LIST_INT"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_for_iter_gen(self):
  def get_items():
  def g():
@@ -671,6 +673,7 @@ def write(items):
  opname = "FOR_ITER_GEN"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_for_iter_list(self):
  def get_items():
  items = []
@@ -692,6 +695,7 @@ def write(items):
  opname = "FOR_ITER_LIST"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_class(self):
  def get_items():
  class C:
@@ -721,6 +725,7 @@ def write(items):
  opname = "LOAD_ATTR_CLASS"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_getattribute_overridden(self):
  def get_items():
  class C:
@@ -750,6 +755,7 @@ def write(items):
  opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_instance_value(self):
  def get_items():
  class C:
@@ -773,6 +779,7 @@ def write(items):
  opname = "LOAD_ATTR_INSTANCE_VALUE"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_method_lazy_dict(self):
  def get_items():
  class C(Exception):
@@ -802,6 +809,7 @@ def write(items):
  opname = "LOAD_ATTR_METHOD_LAZY_DICT"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_method_no_dict(self):
  def get_items():
  class C:
@@ -832,6 +840,7 @@ def write(items):
  opname = "LOAD_ATTR_METHOD_NO_DICT"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_method_with_values(self):
  def get_items():
  class C:
@@ -861,6 +870,7 @@ def write(items):
  opname = "LOAD_ATTR_METHOD_WITH_VALUES"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_module(self):
  def get_items():
  items = []
@@ -885,6 +895,7 @@ def write(items):
  opname = "LOAD_ATTR_MODULE"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_property(self):
  def get_items():
  class C:
@@ -914,6 +925,7 @@ def write(items):
  opname = "LOAD_ATTR_PROPERTY"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_load_attr_with_hint(self):
  def get_items():
  class C:
@@ -940,6 +952,7 @@ def write(items):
  opname = "LOAD_ATTR_WITH_HINT"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization_ft
  def test_load_global_module(self):
  def get_items():
  items = []
@@ -961,6 +974,7 @@ def write(items):
  opname, get_items, read, write, check_items=True
  )
 
+ @requires_specialization
  def test_store_attr_instance_value(self):
  def get_items():
  class C:
@@ -983,6 +997,7 @@ def write(items):
  opname = "STORE_ATTR_INSTANCE_VALUE"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_store_attr_with_hint(self):
  def get_items():
  class C:
@@ -1008,6 +1023,7 @@ def write(items):
  opname = "STORE_ATTR_WITH_HINT"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_store_subscr_list_int(self):
  def get_items():
  items = []
@@ -1031,6 +1047,7 @@ def write(items):
  opname = "STORE_SUBSCR_LIST_INT"
  self.assert_races_do_not_crash(opname, get_items, read, write)
 
+ @requires_specialization
  def test_unpack_sequence_list(self):
  def get_items():
  items = []

Objects/dictobject.c

@@ -1285,6 +1285,20 @@ ensure_shared_on_resize(PyDictObject *mp)
 #endif
 }
 
+static inline void
+ensure_shared_on_keys_version_assignment(PyDictObject *mp)
+{
+ ASSERT_DICT_LOCKED((PyObject *) mp);
+ #ifdef Py_GIL_DISABLED
+ if (!IS_DICT_SHARED(mp)){
+ // This ensures that a concurrent resize operation will delay
+ // freeing the old keys or values using QSBR, which is necessary to
+ // safely allow concurrent reads without locking.
+ SET_DICT_SHARED(mp);
+ }
+ #endif
+}
+
 #ifdef Py_GIL_DISABLED
 
 static inline Py_ALWAYS_INLINE int
@@ -1644,7 +1658,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp,
  }
 
  _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value);
- mp->ma_keys->dk_version = 0;
+FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0);
 
  Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash);
  dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries);
@@ -1686,7 +1700,7 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash)
  ix = unicodekeys_lookup_unicode(keys, key, hash);
  if (ix == DKIX_EMPTY && keys->dk_usable > 0){
  // Insert into new slot
- keys->dk_version = 0;
+FT_ATOMIC_STORE_UINT32_RELAXED(keys->dk_version, 0);
  Py_ssize_t hashpos = find_empty_slot(keys, hash);
  ix = keys->dk_nentries;
  dictkeys_set_index(keys, hashpos, ix);
@@ -2617,7 +2631,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
  ASSERT_CONSISTENT(mp);
  }
  else{
- mp->ma_keys->dk_version = 0;
+FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0);
  dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY);
  if (DK_IS_UNICODE(mp->ma_keys)){
  PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix];
@@ -4429,7 +4443,7 @@ dict_popitem_impl(PyDictObject *self)
  return NULL;
  }
  }
- self->ma_keys->dk_version = 0;
+FT_ATOMIC_STORE_UINT32_RELAXED(self->ma_keys->dk_version, 0);
 
  /* Pop last item */
  PyObject *key, *value;
@@ -7417,20 +7431,54 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys)
  dictkeys_decref(interp, keys, false);
 }
 
-uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp,
- PyDictKeysObject *dictkeys)
+static inline uint32_t
+get_next_dict_keys_version(PyInterpreterState *interp)
 {
- if (dictkeys->dk_version != 0){
- return dictkeys->dk_version;
- }
+#ifdef Py_GIL_DISABLED
+ uint32_t v;
+ do{
+ v = _Py_atomic_load_uint32_relaxed(
+ &interp->dict_state.next_keys_version);
+ if (v == 0){
+ return 0;
+ }
+ } while (!_Py_atomic_compare_exchange_uint32(
+ &interp->dict_state.next_keys_version, &v, v + 1));
+#else
  if (interp->dict_state.next_keys_version == 0){
  return 0;
  }
  uint32_t v = interp->dict_state.next_keys_version++;
- dictkeys->dk_version = v;
+#endif
  return v;
 }
 
+// In free-threaded builds the caller must ensure that the keys object is not
+// being mutated concurrently by another thread.
+uint32_t
+_PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp,
+ PyDictKeysObject *dictkeys)
+{
+ uint32_t dk_version = FT_ATOMIC_LOAD_UINT32_RELAXED(dictkeys->dk_version);
+ if (dk_version != 0){
+ return dk_version;
+ }
+ dk_version = get_next_dict_keys_version(interp);
+ FT_ATOMIC_STORE_UINT32_RELAXED(dictkeys->dk_version, dk_version);
+ return dk_version;
+}
+
+uint32_t
+_PyDict_GetKeysVersionForCurrentState(PyInterpreterState *interp,
+ PyDictObject *dict)
+{
+ ASSERT_DICT_LOCKED((PyObject *) dict);
+ uint32_t dk_version =
+ _PyDictKeys_GetVersionForCurrentState(interp, dict->ma_keys);
+ ensure_shared_on_keys_version_assignment(dict);
+ return dk_version;
+}
+
 static inline int
 validate_watcher_id(PyInterpreterState *interp, int watcher_id)
 {

Objects/funcobject.c

@@ -289,12 +289,14 @@ functions is running.
 
 */
 
+#ifndef Py_GIL_DISABLED
 static inline struct _func_version_cache_item *
 get_cache_item(PyInterpreterState *interp, uint32_t version)
 {
  return interp->func_state.func_version_cache +
  (version % FUNC_VERSION_CACHE_SIZE);
 }
+#endif
 
 void
 _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version)