WIP bpo-44800: Rename _PyInterpreterFrame to _Py_framedata

Doc/whatsnew/3.11.rst

@@ -919,8 +919,8 @@ Porting to Python 3.11
  #if PY_VERSION_HEX < 0x030900B1
  static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame)
 {
- Py_INCREF(frame->f_code);
- return frame->f_code;
+ Py_INCREF(frame->code);
+ return frame->code;
  }
  #endif
 

Include/cpython/ceval.h

@@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *);
  flag was set, else return 0. */
 PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
-PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc);
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _Py_framedata *fdata, int exc);
 
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);

Include/cpython/genobject.h

@@ -28,7 +28,7 @@ extern "C"{
  char prefix##_running_async; \
  /* The frame */ \
  char prefix##_frame_valid; \
-PyObject *prefix##_iframe[1];
+void *prefix##_fdata[1];
 
 typedef struct{
  /* The gi_ prefix is intended to remind of generator-iterator. */

Include/cpython/pystate.h

@@ -48,7 +48,7 @@ typedef struct _PyCFrame{
  */
  int use_tracing;
  /* Pointer to the currently executing frame (it can be NULL) */
- struct _PyInterpreterFrame *current_frame;
+ struct _Py_framedata *current_frame;
  struct _PyCFrame *previous;
 } _PyCFrame;
 
@@ -260,7 +260,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Frame evaluation API */
 
-typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _PyInterpreterFrame *, int);
+typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _Py_framedata *, int);
 
 PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
  PyInterpreterState *interp);

Include/internal/pycore_ceval.h

@@ -47,12 +47,12 @@ extern PyObject *_PyEval_BuiltinsFromGlobals(
 
 
 static inline PyObject*
-_PyEval_EvalFrame(PyThreadState *tstate, struct _PyInterpreterFrame *frame, int throwflag)
+_PyEval_EvalFrame(PyThreadState *tstate, struct _Py_framedata *fdata, int throwflag)
 {
  if (tstate->interp->eval_frame == NULL){
- return _PyEval_EvalFrameDefault(tstate, frame, throwflag);
+ return _PyEval_EvalFrameDefault(tstate, fdata, throwflag);
  }
- return tstate->interp->eval_frame(tstate, frame, throwflag);
+ return tstate->interp->eval_frame(tstate, fdata, throwflag);
 }
 
 extern PyObject *
@@ -116,7 +116,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void){
 
 #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
 
-struct _PyInterpreterFrame *_PyEval_GetFrame(void);
+struct _Py_framedata *_PyEval_GetFrame(void);
 
 PyObject *_Py_MakeCoro(PyFunctionObject *func);
 

Include/internal/pycore_frame.h

@@ -6,17 +6,60 @@ extern "C"{
 
 #include <stdbool.h>
 
+/* Starting in CPython 3.11, CPython separates the frame state between the
+ * full frame objects exposed by the Python and C runtime state introspection
+ * APIs, and internal lighter weight frame data structs, which are simple C
+ * structures owned by either the interpreter eval loop (while executing
+ * ordinary functions), by a generator or coroutine object (for frames that
+ * are able to be suspended), or by their corresponding full frame object (if
+ * a state instrospection API has been invoked and the full frame object has
+ * taken responsibility for the lifecycle of the frame data storage).
+ *
+ * This split storage eliminates a lot of allocation and deallocation of full
+ * Python objects during code execution, providing a significant speed gain
+ * over the previous approach of using full Python objects for both
+ * introspection and code execution.
+ *
+ * Field naming conventions:
+ *
+ * * full frame object fields have an "f_*" prefix
+ * * frame data struct fields have no prefix
+ *
+ * Local variable and function argument naming conventions:
+ *
+ * * "frame", "f", and "frameobj" are used for full frame objects
+ * * Exception: "current_frame" in the thread state cframe struct is a frame data struct
+ * * "fdata" is used for frame data structs
+ *
+ * Function/macro naming conventions:
+ *
+ * * "PyFrame_*" functions accept a full frame object
+ * * "_PyFrame_*" functions accept either a full frame object or a frame
+ * data struct. Check the specific function signatures for details.
+ * * Other public C API functions that relate to frames only accept full
+ * frame objects
+ * * Other private C API functions that relate to frames may accept either a
+ * full frame object or a frame data struct. Check the specific function
+ * signatures for details
+ *
+ * Function return types:
+ * * Public C API functions will only ever return full frame objects
+ * * Private C API functions with an underscore prefix may return frame
+ * data structs instead
+ */
+
+
 struct _frame{
  PyObject_HEAD
  PyFrameObject *f_back; /* previous frame, or NULL */
- struct _PyInterpreterFrame *f_frame; /* points to the frame data */
+ struct _Py_framedata *f_fdata; /* points to the frame data */
  PyObject *f_trace; /* Trace function */
  int f_lineno; /* Current line number. Only valid if non-zero */
  char f_trace_lines; /* Emit per-line trace events? */
  char f_trace_opcodes; /* Emit per-opcode trace events? */
  char f_owns_frame; /* This frame owns the frame */
  /* The frame data, if this frame object owns the frame */
-PyObject *_f_frame_data[1];
+struct _Py_framedata *_f_owned_fdata[1];
 };
 
 extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code);
@@ -29,7 +72,8 @@ extern void _PyFrame_Fini(PyInterpreterState *interp);
 /* other API */
 
 /* These values are chosen so that the inline functions below all
- * compare f_state to zero.
+ * compare fdata->state to zero while keeping the property that most
+ * state transitions move to a higher value frame state.
  */
 enum _framestate{
  FRAME_CREATED = -2,
@@ -44,146 +88,146 @@ enum _framestate{
 typedef signed char PyFrameState;
 
 /*
- frame->f_lasti refers to the index of the last instruction,
+ frame->lasti refers to the index of the last instruction,
  unless it's -1 in which case next_instr should be first_instr.
 */
 
-typedef struct _PyInterpreterFrame{
- PyFunctionObject *f_func; /* Strong reference */
- PyObject *f_globals; /* Borrowed reference */
- PyObject *f_builtins; /* Borrowed reference */
- PyObject *f_locals; /* Strong reference, may be NULL */
- PyCodeObject *f_code; /* Strong reference */
+typedef struct _Py_framedata{
+ PyFunctionObject *func; /* Strong reference */
+ PyObject *globals; /* Borrowed reference */
+ PyObject *builtins; /* Borrowed reference */
+ PyObject *locals; /* Strong reference, may be NULL */
+ PyCodeObject *code; /* Strong reference */
  PyFrameObject *frame_obj; /* Strong reference, may be NULL */
- struct _PyInterpreterFrame *previous;
- int f_lasti; /* Last instruction if called */
+ struct _Py_framedata *previous;
+ int lasti; /* Last instruction if called */
  int stacktop; /* Offset of TOS from localsplus */
- PyFrameState f_state; /* What state the frame is in */
+ PyFrameState state; /* What state the frame is in */
  bool is_entry; // Whether this is the "root" frame for the current _PyCFrame.
  bool is_generator;
  PyObject *localsplus[1];
-} _PyInterpreterFrame;
+} _Py_framedata;
 
-static inline int _PyFrame_IsRunnable(_PyInterpreterFrame *f){
- return f->f_state < FRAME_EXECUTING;
+static inline int _PyFrame_IsRunnable(_Py_framedata *f){
+ return f->state < FRAME_EXECUTING;
 }
 
-static inline int _PyFrame_IsExecuting(_PyInterpreterFrame *f){
- return f->f_state == FRAME_EXECUTING;
+static inline int _PyFrame_IsExecuting(_Py_framedata *f){
+ return f->state == FRAME_EXECUTING;
 }
 
-static inline int _PyFrameHasCompleted(_PyInterpreterFrame *f){
- return f->f_state > FRAME_EXECUTING;
+static inline int _PyFrameHasCompleted(_Py_framedata *f){
+ return f->state > FRAME_EXECUTING;
 }
 
-static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f){
- return f->localsplus + f->f_code->co_nlocalsplus;
+static inline PyObject **_PyFrame_Stackbase(_Py_framedata *f){
+ return f->localsplus + f->code->co_nlocalsplus;
 }
 
-static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f){
- assert(f->stacktop > f->f_code->co_nlocalsplus);
+static inline PyObject *_PyFrame_StackPeek(_Py_framedata *f){
+ assert(f->stacktop > f->code->co_nlocalsplus);
  assert(f->localsplus[f->stacktop-1] != NULL);
  return f->localsplus[f->stacktop-1];
 }
 
-static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f){
- assert(f->stacktop > f->f_code->co_nlocalsplus);
+static inline PyObject *_PyFrame_StackPop(_Py_framedata *f){
+ assert(f->stacktop > f->code->co_nlocalsplus);
  f->stacktop--;
  return f->localsplus[f->stacktop];
 }
 
-static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value){
-f->localsplus[f->stacktop] = value;
-f->stacktop++;
+static inline void _PyFrame_StackPush(_Py_framedata *fdata, PyObject *value){
+fdata->localsplus[fdata->stacktop] = value;
+fdata->stacktop++;
 }
 
-#define FRAME_SPECIALS_SIZE ((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))
+#define FRAME_SPECIALS_SIZE ((sizeof(_Py_framedata)-1)/sizeof(PyObject *))
 
-void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest);
+void _PyFrame_Copy(_Py_framedata *src, _Py_framedata *dest);
 
 /* Consumes reference to func */
 static inline void
 _PyFrame_InitializeSpecials(
-_PyInterpreterFrame *frame, PyFunctionObject *func,
+_Py_framedata *fdata, PyFunctionObject *func,
  PyObject *locals, int nlocalsplus)
 {
-frame->f_func = func;
-frame->f_code = (PyCodeObject *)Py_NewRef(func->func_code);
-frame->f_builtins = func->func_builtins;
-frame->f_globals = func->func_globals;
-frame->f_locals = Py_XNewRef(locals);
-frame->stacktop = nlocalsplus;
-frame->frame_obj = NULL;
-frame->f_lasti = -1;
-frame->f_state = FRAME_CREATED;
-frame->is_entry = false;
-frame->is_generator = false;
+fdata->func = func;
+fdata->code = (PyCodeObject *)Py_NewRef(func->func_code);
+fdata->builtins = func->func_builtins;
+fdata->globals = func->func_globals;
+fdata->locals = Py_XNewRef(locals);
+fdata->stacktop = nlocalsplus;
+fdata->frame_obj = NULL;
+fdata->lasti = -1;
+fdata->state = FRAME_CREATED;
+fdata->is_entry = false;
+fdata->is_generator = false;
 }
 
 /* Gets the pointer to the locals array
  * that precedes this frame.
  */
 static inline PyObject**
-_PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
+_PyFrame_GetLocalsArray(_Py_framedata *fdata)
 {
- return frame->localsplus;
+ return fdata->localsplus;
 }
 
 static inline PyObject**
-_PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
+_PyFrame_GetStackPointer(_Py_framedata *fdata)
 {
- return frame->localsplus+frame->stacktop;
+ return fdata->localsplus+fdata->stacktop;
 }
 
 static inline void
-_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
+_PyFrame_SetStackPointer(_Py_framedata *fdata, PyObject **stack_pointer)
 {
-frame->stacktop = (int)(stack_pointer - frame->localsplus);
+fdata->stacktop = (int)(stack_pointer - fdata->localsplus);
 }
 
 /* For use by _PyFrame_GetFrameObject
  Do not call directly. */
 PyFrameObject *
-_PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame);
+_PyFrame_MakeAndSetFrameObject(_Py_framedata *fdata);
 
 /* Gets the PyFrameObject for this frame, lazily
  * creating it if necessary.
  * Returns a borrowed referennce */
 static inline PyFrameObject *
-_PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
+_PyFrame_GetFrameObject(_Py_framedata *fdata)
 {
- PyFrameObject *res =  frame->frame_obj;
+ PyFrameObject *res = fdata->frame_obj;
  if (res != NULL){
  return res;
  }
- return _PyFrame_MakeAndSetFrameObject(frame);
+ return _PyFrame_MakeAndSetFrameObject(fdata);
 }
 
 /* Clears all references in the frame.
- * If take is non-zero, then the _PyInterpreterFrame frame
- * may be transferred to the frame object it references
+ * If take is non-zero, then the frame data
+ * may be transfered to the frame object it references
  * instead of being cleared. Either way
  * the caller no longer owns the references
  * in the frame.
- * take should be set to 1 for heap allocated
+ * take should be set to 1 for heap allocated
  * frames like the ones in generators and coroutines.
  */
 void
-_PyFrame_Clear(_PyInterpreterFrame * frame);
+_PyFrame_Clear(_Py_framedata * fdata);
 
 int
-_PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg);
+_PyFrame_Traverse(_Py_framedata *fdata, visitproc visit, void *arg);
 
 int
-_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame);
+_PyFrame_FastToLocalsWithError(_Py_framedata *fdata);
 
 void
-_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear);
+_PyFrame_LocalsToFast(_Py_framedata *fdata, int clear);
 
-extern _PyInterpreterFrame *
+extern _Py_framedata *
 _PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size);
 
-static inline _PyInterpreterFrame *
+static inline _Py_framedata *
 _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
 {
  PyObject **base = tstate->datastack_top;
@@ -192,16 +236,16 @@ _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size)
  assert(tstate->datastack_limit);
  if (top < tstate->datastack_limit){
  tstate->datastack_top = top;
- return (_PyInterpreterFrame *)base;
+ return (_Py_framedata *)base;
  }
  }
  return _PyThreadState_BumpFramePointerSlow(tstate, size);
 }
 
-void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame);
+void _PyThreadState_PopFrame(PyThreadState *tstate, _Py_framedata *fdata);
 
 /* Consume reference to func */
-_PyInterpreterFrame *
+_Py_framedata *
 _PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func);
 
 #ifdef __cplusplus

Misc/NEWS.d/next/Core and Builtins/2021-08-01-18-18-51.bpo-44800.TCsfH3.rst

@@ -0,0 +1,8 @@
+Refactored internal APIs for the new lazy frame object creation to more
+consistently distinguish between the full ``PyFrameObject`` Python object
+implementation that is still used in the Python and C runtime state
+introspection APIs (function prefix ``PyFrame``, field prefix ``f_``, typical
+variable names ``frame`` and ``f``) and the new ``_Py_framedata`` internal
+frame data storage (C structs with no intrinsic instance lifecycle management)
+that is now used for code execution (function prefix ``_Py_framedata``, no
+field prefix, typical variable name ``fdata``).