gh-98003: Inline call frames for CALL_FUNCTION_EX

Include/internal/pycore_call.h

@@ -116,6 +116,16 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const
  return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL);
 }
 
+PyObject *const *
+_PyStack_UnpackDict(PyThreadState *tstate,
+ PyObject *const *args, Py_ssize_t nargs,
+ PyObject *kwargs, PyObject **p_kwnames);
+
+void
+_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs,
+ PyObject *kwnames);
+
+void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames);
 
 #ifdef __cplusplus
 }

Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst

@@ -0,0 +1,3 @@
+Complex function calls are now faster and consume no C stack
+space.
+

Objects/call.c

@@ -8,16 +8,6 @@
 #include "pycore_tuple.h" // _PyTuple_ITEMS()
 
 
-static PyObject *const *
-_PyStack_UnpackDict(PyThreadState *tstate,
- PyObject *const *args, Py_ssize_t nargs,
- PyObject *kwargs, PyObject **p_kwnames);
-
-static void
-_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs,
- PyObject *kwnames);
-
-
 static PyObject *
 null_error(PyThreadState *tstate)
 {
@@ -965,7 +955,7 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames)
  The newly allocated argument vector supports PY_VECTORCALL_ARGUMENTS_OFFSET.
 
  When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */
-static PyObject *const *
+PyObject *const *
 _PyStack_UnpackDict(PyThreadState *tstate,
  PyObject *const *args, Py_ssize_t nargs,
  PyObject *kwargs, PyObject **p_kwnames)
@@ -1034,14 +1024,20 @@ _PyStack_UnpackDict(PyThreadState *tstate,
  return stack;
 }
 
-static void
+void
 _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs,
  PyObject *kwnames)
 {
  Py_ssize_t n = PyTuple_GET_SIZE(kwnames) + nargs;
  for (Py_ssize_t i = 0; i < n; i++){
  Py_DECREF(stack[i]);
  }
+ _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames);
+}
+
+void
+_PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames)
+{
  PyMem_Free((PyObject **)stack - 1);
  Py_DECREF(kwnames);
 }

Python/bytecodes.c

@@ -3103,6 +3103,25 @@ dummy_func(
  }
  }
  else{
+ if (Py_TYPE(func) == &PyFunction_Type &&
+ tstate->interp->eval_frame == NULL &&
+ ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall){
+ assert(PyTuple_CheckExact(callargs));
+ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
+ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
+
+ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
+ (PyFunctionObject *)func, locals,
+ nargs, callargs, kwargs);
+ // Need to manually shrink the stack since we exit with DISPATCH_INLINED.
+ STACK_SHRINK(oparg + 3);
+ if (new_frame == NULL){
+ goto error;
+ }
+ frame->return_offset = 0;
+ DISPATCH_INLINED(new_frame);
+ }
  result = PyObject_Call(func, callargs, kwargs);
  }
  DECREF_INPUTS();

Python/ceval.c

@@ -212,6 +212,9 @@ static _PyInterpreterFrame *
 _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
  PyObject *locals, PyObject* const* args,
  size_t argcount, PyObject *kwnames);
+static _PyInterpreterFrame *
+_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
+ PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs);
 static void
 _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
 
@@ -1501,6 +1504,49 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
  return NULL;
 }
 
+/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict.
+ Steals references to func, callargs and kwargs.
+*/
+static _PyInterpreterFrame *
+_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
+ PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs)
+{
+ bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0);
+ PyObject *kwnames = NULL;
+ PyObject *const *newargs;
+ if (has_dict){
+ newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames);
+ if (newargs == NULL){
+ Py_DECREF(func);
+ goto error;
+ }
+ }
+ else{
+ newargs = &PyTuple_GET_ITEM(callargs, 0);
+ /* We need to incref all our args since the new frame steals the references. */
+ for (Py_ssize_t i = 0; i < nargs; ++i){
+ Py_INCREF(PyTuple_GET_ITEM(callargs, i));
+ }
+ }
+ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
+ tstate, (PyFunctionObject *)func, locals,
+ newargs, nargs, kwnames
+ );
+ if (has_dict){
+ _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames);
+ }
+ /* No need to decref func here because the reference has been stolen by
+ _PyEvalFramePushAndInit.
+ */
+ Py_DECREF(callargs);
+ Py_XDECREF(kwargs);
+ return new_frame;
+error:
+ Py_DECREF(callargs);
+ Py_XDECREF(kwargs);
+ return NULL;
+}
+
 PyObject *
 _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
  PyObject *locals,