gh-91052: Add C API for watching dictionaries

Doc/c-api/dict.rst

@@ -238,3 +238,54 @@ Dictionary Objects
  for key, value in seq2:
  if override or key not in a:
  a[key] = value
+
+.. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback)
+
+ Register *callback* as a dictionary watcher. Return a non-negative integer
+ id which must be passed to future calls to :c:func:`PyDict_Watch`. In case
+ of error (e.g. no more watcher IDs available), return ``-1`` and set an
+ exception.
+
+.. c:function:: int PyDict_ClearWatcher(int watcher_id)
+
+ Clear watcher identified by *watcher_id* previously returned from
+ :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g.
+ if the given *watcher_id* was never registered.)
+
+.. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict)
+
+ Mark dictionary *dict* as watched. The callback granted *watcher_id* by
+ :c:func:`PyDict_AddWatcher` will be called when *dict* is modified or
+ deallocated.
+
+.. c:type:: PyDict_WatchEvent
+
+ Enumeration of possible dictionary watcher events: ``PyDict_EVENT_ADDED``,
+ ``PyDict_EVENT_MODIFIED``, ``PyDict_EVENT_DELETED``, ``PyDict_EVENT_CLONED``,
+ ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCATED``.
+
+.. c:type:: int (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value)
+
+ Type of a dict watcher callback function.
+
+ If *event* is ``PyDict_EVENT_CLEARED`` or ``PyDict_EVENT_DEALLOCATED``, both
+ *key* and *new_value* will be ``NULL``. If *event* is ``PyDict_EVENT_ADDED``
+ or ``PyDict_EVENT_MODIFIED``, *new_value* will be the new value for *key*.
+ If *event* is ``PyDict_EVENT_DELETED``, *key* is being deleted from the
+ dictionary and *new_value* will be ``NULL``.
+
+ ``PyDict_EVENT_CLONED`` occurs when *dict* was previously empty and another
+ dict is merged into it. To maintain efficiency of this operation, per-key
+ ``PyDict_EVENT_ADDED`` events are not issued in this case; instead a
+ single ``PyDict_EVENT_CLONED`` is issued, and *key* will be the source
+ dictionary.
+
+ The callback may inspect but must not modify *dict*; doing so could have
+ unpredictable effects, including infinite recursion.
+
+ Callbacks occur before the notified modification to *dict* takes place, so
+ the prior state of *dict* can be inspected.
+
+ If the callback returns with an exception set, it must return ``-1``; this
+ exception will be printed as an unraisable exception using
+ :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.

Include/cpython/dictobject.h

@@ -83,3 +83,26 @@ typedef struct{
 
 PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other);
+
+/* Dictionary watchers */
+
+typedef enum{
+ PyDict_EVENT_ADDED,
+ PyDict_EVENT_MODIFIED,
+ PyDict_EVENT_DELETED,
+ PyDict_EVENT_CLONED,
+ PyDict_EVENT_CLEARED,
+ PyDict_EVENT_DEALLOCATED,
+} PyDict_WatchEvent;
+
+// Callback to be invoked when a watched dict is cleared, dealloced, or modified.
+// In clear/dealloc case, key and new_value will be NULL. Otherwise, new_value will be the
+// new value for key, NULL if key is being deleted.
+typedef int(*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value);
+
+// Register/unregister a dict-watcher callback
+PyAPI_FUNC(int) PyDict_AddWatcher(PyDict_WatchCallback callback);
+PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id);
+
+// Mark given dictionary as "watched" (callback will be called if it is modified)
+PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict);

Include/internal/pycore_dict.h

@@ -154,7 +154,32 @@ struct _dictvalues{
 
 extern uint64_t _pydict_global_version;
 
-#define DICT_NEXT_VERSION() (++_pydict_global_version)
+#define DICT_MAX_WATCHERS 8
+#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS)
+#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1)
+
+#define DICT_NEXT_VERSION() (_pydict_global_version += DICT_VERSION_INCREMENT)
+
+void
+_PyDict_SendEvent(int watcher_bits,
+ PyDict_WatchEvent event,
+ PyDictObject *mp,
+ PyObject *key,
+ PyObject *value);
+
+static inline uint64_t
+_PyDict_NotifyEvent(PyDict_WatchEvent event,
+ PyDictObject *mp,
+ PyObject *key,
+ PyObject *value)
+{
+ int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK;
+ if (watcher_bits){
+ _PyDict_SendEvent(watcher_bits, event, mp, key, value);
+ return DICT_NEXT_VERSION() | watcher_bits;
+ }
+ return DICT_NEXT_VERSION();
+}
 
 extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
 extern PyObject *_PyDict_FromItems(

Include/internal/pycore_interp.h

@@ -144,6 +144,8 @@ struct _is{
  // Initialized to _PyEval_EvalFrameDefault().
  _PyFrameEvalFunction eval_frame;
 
+ PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS];
+
  Py_ssize_t co_extra_user_count;
  freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 

Lib/test/test_capi.py

@@ -2,6 +2,7 @@
 # these are all functions _testcapi exports whose name begins with 'test_'.
 
 from collections import OrderedDict
+from contextlib import contextmanager
 import _thread
 import importlib.machinery
 import importlib.util
@@ -1393,5 +1394,136 @@ def func2(x=None):
  self.do_test(func2)
 
 
+class TestDictWatchers(unittest.TestCase):
+ # types of watchers testcapimodule can add:
+ EVENTS = 0 # appends dict events as strings to global event list
+ ERROR = 1 # unconditionally sets and signals a RuntimeException
+ SECOND = 2 # always appends "second" to global event list
+
+ def add_watcher(self, kind=EVENTS):
+ return _testcapi.add_dict_watcher(kind)
+
+ def clear_watcher(self, watcher_id):
+ _testcapi.clear_dict_watcher(watcher_id)
+
+ @contextmanager
+ def watcher(self, kind=EVENTS):
+ wid = self.add_watcher(kind)
+ try:
+ yield wid
+ finally:
+ self.clear_watcher(wid)
+
+ def assert_events(self, expected):
+ actual = _testcapi.get_dict_watcher_events()
+ self.assertEqual(actual, expected)
+
+ def watch(self, wid, d):
+ _testcapi.watch_dict(wid, d)
+
+ def test_set_new_item(self):
+ d ={}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ d["foo"] = "bar"
+ self.assert_events(["new:foo:bar"])
+
+ def test_set_existing_item(self):
+ d ={"foo": "bar"}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ d["foo"] = "baz"
+ self.assert_events(["mod:foo:baz"])
+
+ def test_clone(self):
+ d ={}
+ d2 ={"foo": "bar"}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ d.update(d2)
+ self.assert_events(["clone"])
+
+ def test_no_event_if_not_watched(self):
+ d ={}
+ with self.watcher() as wid:
+ d["foo"] = "bar"
+ self.assert_events([])
+
+ def test_del(self):
+ d ={"foo": "bar"}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ del d["foo"]
+ self.assert_events(["del:foo"])
+
+ def test_pop(self):
+ d ={"foo": "bar"}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ d.pop("foo")
+ self.assert_events(["del:foo"])
+
+ def test_clear(self):
+ d ={"foo": "bar"}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ d.clear()
+ self.assert_events(["clear"])
+
+ def test_dealloc(self):
+ d ={"foo": "bar"}
+ with self.watcher() as wid:
+ self.watch(wid, d)
+ del d
+ self.assert_events(["dealloc"])
+
+ def test_error(self):
+ d ={}
+ unraisables = []
+ def unraisable_hook(unraisable):
+ unraisables.append(unraisable)
+ with self.watcher(kind=self.ERROR) as wid:
+ self.watch(wid, d)
+ orig_unraisable_hook = sys.unraisablehook
+ sys.unraisablehook = unraisable_hook
+ try:
+ d["foo"] = "bar"
+ finally:
+ sys.unraisablehook = orig_unraisable_hook
+ self.assert_events([])
+ self.assertEqual(len(unraisables), 1)
+ unraisable = unraisables[0]
+ self.assertIs(unraisable.object, d)
+ self.assertEqual(str(unraisable.exc_value), "boom!")
+
+ def test_two_watchers(self):
+ d1 ={}
+ d2 ={}
+ with self.watcher() as wid1:
+ with self.watcher(kind=self.SECOND) as wid2:
+ self.watch(wid1, d1)
+ self.watch(wid2, d2)
+ d1["foo"] = "bar"
+ d2["hmm"] = "baz"
+ self.assert_events(["new:foo:bar", "second"])
+
+ def test_watch_non_dict(self):
+ with self.watcher() as wid:
+ with self.assertRaisesRegex(ValueError, r"Cannot watch non-dictionary"):
+ self.watch(wid, 1)
+
+ def test_watch_out_of_range_watcher_id(self):
+ d ={}
+ with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"):
+ self.watch(-1, d)
+ with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID 8"):
+ self.watch(8, d) # DICT_MAX_WATCHERS = 8
+
+ def test_unassigned_watcher_id(self):
+ d ={}
+ with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"):
+ self.watch(1, d)
+
+
 if __name__ == "__main__":
  unittest.main()

Misc/NEWS.d/next/Core and Builtins/2022-10-03-16-12-39.gh-issue-91052.MsYL9d.rst

@@ -0,0 +1 @@
+Add API for subscribing to modification events on selected dictionaries.