gh-76785: Improved Subinterpreters Compatibility with 3.12

Include/internal/pycore_code.h

@@ -8,6 +8,15 @@ extern "C"{
 # error "this header requires Py_BUILD_CORE define"
 #endif
 
+
+// We hide some of the newer PyCodeObject fields behind macros.
+// This helps with backporting certain changes to 3.12.
+#define _PyCode_HAS_EXECUTORS(CODE) \
+ (CODE->co_executors != NULL)
+#define _PyCode_HAS_INSTRUMENTATION(CODE) \
+ (CODE->_co_instrumentation_version > 0)
+
+
 #define CODE_MAX_WATCHERS 8
 
 /* PEP 659

Include/internal/pycore_crossinterp.h

@@ -87,6 +87,11 @@ struct _xid{
 PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void);
 PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data);
 
+#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data)
+#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj)
+#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid)
+// Users should not need getters for "new_object" or "free".
+
 
 /* defining cross-interpreter data */
 
@@ -101,6 +106,25 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
 PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
  PyInterpreterState *, _PyCrossInterpreterData *);
 
+// Normally the Init* functions are sufficient. The only time
+// additional initialization might be needed is to set the "free" func,
+// though that should be infrequent.
+#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \
+ do{\
+ (DATA)->free = (FUNC); \
+ } while (0)
+// Additionally, some shareable types are essentially light wrappers
+// around other shareable types. The crossinterpdatafunc of the wrapper
+// can often be implemented by calling the wrapped object's
+// crossinterpdatafunc and then changing the "new_object" function.
+// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that,
+// but might be better to have a function like
+// _PyCrossInterpreterData_AdaptToWrapper() instead.
+#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \
+ do{\
+ (DATA)->new_object = (FUNC); \
+ } while (0)
+
 
 /* using cross-interpreter data */
 
@@ -170,6 +194,8 @@ extern void _PyXI_Fini(PyInterpreterState *interp);
 extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp);
 extern void _PyXI_FiniTypes(PyInterpreterState *interp);
 
+#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi)
+
 
 /***************************/
 /* short-term data sharing */

Include/internal/pycore_tstate.h

@@ -13,6 +13,13 @@ extern "C"{
 #include "pycore_brc.h" // struct _brc_thread_state
 
 
+static inline void
+_PyThreadState_SetWhence(PyThreadState *tstate, int whence)
+{
+ tstate->_whence = whence;
+}
+
+
 // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
 // PyThreadState fields are exposed as part of the C API, although most fields
 // are intended to be private. The _PyThreadStateImpl fields not exposed.

Makefile.pre.in

@@ -1671,6 +1671,14 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule
 
 Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h
 
+Modules/_xxsubinterpretersmodule.o: $(srcdir)/Modules/_xxsubinterpretersmodule.c $(srcdir)/Modules/_interpreters_common.h
+
+Modules/_xxinterpqueuesmodule.o: $(srcdir)/Modules/_xxinterpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h
+
+Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h
+
+Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h
+
 Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile
  $(CC) -c $(PY_CORE_CFLAGS) \
  -DSOABI='"$(SOABI)"' \

Modules/_interpreters_common.h

@@ -0,0 +1,13 @@
+
+#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \
+ PyInit_ ## NAME
+#define RESOLVE_MODINIT_FUNC_NAME(NAME) \
+ _RESOLVE_MODINIT_FUNC_NAME(NAME)
+
+
+static int
+ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata)
+{
+ //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
+ return _PyCrossInterpreterData_RegisterClass(cls, getdata);
+}

Modules/_xxinterpchannelsmodule.c

@@ -17,6 +17,8 @@
 #include <sched.h> // sched_yield()
 #endif
 
+#include "_interpreters_common.h"
+
 
 /*
 This module has the following process-global state:
@@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*()
 API.. The module does not create any objects that are shared globally.
 */
 
-#define MODULE_NAME "_xxinterpchannels"
+#define MODULE_NAME _xxinterpchannels
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 #define GLOBAL_MALLOC(TYPE) \
@@ -101,7 +105,7 @@ static int
 register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared,
  struct xid_class_registry *classes)
 {
- int res = _PyCrossInterpreterData_RegisterClass(cls, shared);
+ int res = ensure_xid_class(cls, shared);
  if (res == 0){
  assert(classes->count < MAX_XID_CLASSES);
  // The class has refs elsewhere, so we need to incref here.
@@ -167,7 +171,7 @@ _get_current_interp(void)
 static PyObject *
 _get_current_module(void)
 {
- PyObject *name = PyUnicode_FromString(MODULE_NAME);
+ PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
  if (name == NULL){
  return NULL;
  }
@@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
 }
 
 #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \
- add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
+ add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE)
 
 static PyTypeObject *
 add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared,
@@ -299,7 +303,7 @@ _get_current_module_state(void)
  if (mod == NULL){
  // XXX import it?
  PyErr_SetString(PyExc_RuntimeError,
-MODULE_NAME " module not imported yet");
+MODULE_NAME_STR " module not imported yet");
  return NULL;
  }
  module_state *state = get_module_state(mod);
@@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid)
  while (next != NULL){
  _channelitem *item = next;
  next = item->next;
- if (item->data->interpid == interpid){
+ if (_PyCrossInterpreterData_INTERPID(item->data) == interpid){
  if (prev == NULL){
  queue->first = item->next;
  }
@@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] ={
 };
 
 static PyStructSequence_Desc channel_info_desc ={
- .name = MODULE_NAME ".ChannelInfo",
+ .name = MODULE_NAME_STR ".ChannelInfo",
  .doc = channel_info_doc,
  .fields = channel_info_fields,
  .n_in_sequence = 8,
@@ -2474,10 +2478,11 @@ struct _channelid_xid{
 static PyObject *
 _channelid_from_xid(_PyCrossInterpreterData *data)
 {
- struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
+ struct _channelid_xid *xid = \
+ (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
 
  // It might not be imported yet, so we can't use _get_current_module().
- PyObject *mod = PyImport_ImportModule(MODULE_NAME);
+ PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR);
  if (mod == NULL){
  return NULL;
  }
@@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj,
 {
  return -1;
  }
- struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
+ struct _channelid_xid *xid = \
+ (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
  xid->cid = ((channelid *)obj)->cid;
  xid->end = ((channelid *)obj)->end;
  xid->resolve = ((channelid *)obj)->resolve;
@@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] ={
 };
 
 static PyType_Spec channelid_typespec ={
- .name = MODULE_NAME ".ChannelID",
+ .name = MODULE_NAME_STR ".ChannelID",
  .basicsize = sizeof(channelid),
  .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
  Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj,
  if (res < 0){
  return -1;
  }
- data->new_object = _channelend_from_xid;
+_PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid);
  return 0;
 }
 
@@ -3379,7 +3385,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef ={
  .m_base = PyModuleDef_HEAD_INIT,
- .m_name = MODULE_NAME,
+ .m_name = MODULE_NAME_STR,
  .m_doc = module_doc,
  .m_size = sizeof(module_state),
  .m_methods = module_functions,
@@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef ={
 };
 
 PyMODINIT_FUNC
-PyInit__xxinterpchannels(void)
+MODINIT_FUNC_NAME(void)
 {
  return PyModuleDef_Init(&moduledef);
 }

Modules/_xxinterpqueuesmodule.c

@@ -8,8 +8,12 @@
 #include "Python.h"
 #include "pycore_crossinterp.h" // struct _xid
 
+#include "_interpreters_common.h"
 
-#define MODULE_NAME "_xxinterpqueues"
+
+#define MODULE_NAME _xxinterpqueues
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 #define GLOBAL_MALLOC(TYPE) \
@@ -64,7 +68,7 @@ _get_current_interp(void)
 static PyObject *
 _get_current_module(void)
 {
- PyObject *name = PyUnicode_FromString(MODULE_NAME);
+ PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
  if (name == NULL){
  return NULL;
  }
@@ -602,7 +606,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid)
  while (next != NULL){
  _queueitem *item = next;
  next = item->next;
- if (item->data->interpid == interpid){
+ if (_PyCrossInterpreterData_INTERPID(item->data) == interpid){
  if (prev == NULL){
  queue->items.first = item->next;
  }
@@ -1062,7 +1066,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type)
  }
  state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);
 
- if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0){
+ if (ensure_xid_class(queue_type, _queueobj_shared) < 0){
  return -1;
  }
 
@@ -1130,7 +1134,7 @@ _queueid_xid_free(void *data)
 static PyObject *
 _queueobj_from_xid(_PyCrossInterpreterData *data)
 {
- int64_t qid = *(int64_t *)data->data;
+ int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data);
  PyObject *qidobj = PyLong_FromLongLong(qid);
  if (qidobj == NULL){
  return NULL;
@@ -1140,7 +1144,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data)
  if (mod == NULL){
  // XXX import it?
  PyErr_SetString(PyExc_RuntimeError,
-MODULE_NAME " module not imported yet");
+MODULE_NAME_STR " module not imported yet");
  return NULL;
  }
 
@@ -1181,7 +1185,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj,
  _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL,
  _queueobj_from_xid);
  Py_DECREF(qidobj);
- data->free = _queueid_xid_free;
+_PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free);
  return 0;
 }
 
@@ -1670,7 +1674,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef ={
  .m_base = PyModuleDef_HEAD_INIT,
- .m_name = MODULE_NAME,
+ .m_name = MODULE_NAME_STR,
  .m_doc = module_doc,
  .m_size = sizeof(module_state),
  .m_methods = module_functions,
@@ -1681,7 +1685,7 @@ static struct PyModuleDef moduledef ={
 };
 
 PyMODINIT_FUNC
-PyInit__xxinterpqueues(void)
+MODINIT_FUNC_NAME(void)
 {
  return PyModuleDef_Init(&moduledef);
 }