bpo-46409: Make generators in bytecode

Doc/library/dis.rst

@@ -942,6 +942,13 @@ All of the following opcodes use their arguments.
  Set bytecode counter to *target*.
 
 
+.. opcode:: JUMP_NO_INTERRUPT (target)
+
+ Set bytecode counter to *target*. Do not check for interrupts.
+
+ .. versionadded:: 3.11
+
+
 .. opcode:: FOR_ITER (delta)
 
  TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If
@@ -1220,6 +1227,14 @@ All of the following opcodes use their arguments.
  .. versionadded:: 3.11
 
 
+.. opcode:: RETURN_GENERATOR
+
+ Create a generator, coroutine, or async generator from the current frame.
+ Clear the current frame and return the newly created generator.
+
+ .. versionadded:: 3.11
+
+
 .. opcode:: HAVE_ARGUMENT
 
  This is not really an opcode. It identifies the dividing line between

Include/cpython/genobject.h

@@ -13,7 +13,6 @@ extern "C"{
  and coroutine objects. */
 #define _PyGenObject_HEAD(prefix) \
  PyObject_HEAD \
- /* Note: gi_frame can be NULL if the generator is "finished" */ \
  /* The code object backing the generator */ \
  PyCodeObject *prefix##_code; \
  /* List of weak reference. */ \

Include/internal/pycore_frame.h

@@ -41,12 +41,12 @@ typedef struct _interpreter_frame{
  PyObject *f_locals; /* Strong reference, may be NULL */
  PyCodeObject *f_code; /* Strong reference */
  PyFrameObject *frame_obj; /* Strong reference, may be NULL */
- PyObject *generator; /* Borrowed reference, may be NULL */
  struct _interpreter_frame *previous;
  int f_lasti; /* Last instruction if called */
  int stacktop; /* Offset of TOS from localsplus */
  PyFrameState f_state; /* What state the frame is in */
  bool is_entry; // Whether this is the "root" frame for the current CFrame.
+ bool is_generator;
  PyObject *localsplus[1];
 } InterpreterFrame;
 
@@ -100,10 +100,10 @@ _PyFrame_InitializeSpecials(
  frame->f_locals = Py_XNewRef(locals);
  frame->stacktop = nlocalsplus;
  frame->frame_obj = NULL;
- frame->generator = NULL;
  frame->f_lasti = -1;
  frame->f_state = FRAME_CREATED;
  frame->is_entry = false;
+ frame->is_generator = false;
 }
 
 /* Gets the pointer to the locals array

Lib/importlib/_bootstrap_external.py

@@ -380,6 +380,7 @@ def _write_atomic(path, data, mode=0o666):
 # Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
 # Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
 # Python 3.11a4 3474 (Add RESUME opcode)
+# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
 
 # Python 3.12 will start with magic number 3500
 
@@ -393,7 +394,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3474).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3475).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
 
 _PYCACHE = '__pycache__'

Lib/inspect.py

@@ -1819,11 +1819,11 @@ def getgeneratorstate(generator):
  """
  if generator.gi_running:
  return GEN_RUNNING
+ if generator.gi_suspended:
+ return GEN_SUSPENDED
  if generator.gi_frame is None:
  return GEN_CLOSED
- if generator.gi_frame.f_lasti == -1:
- return GEN_CREATED
- return GEN_SUSPENDED
+ return GEN_CREATED
 
 
 def getgeneratorlocals(generator):
@@ -1861,11 +1861,11 @@ def getcoroutinestate(coroutine):
  """
  if coroutine.cr_running:
  return CORO_RUNNING
+ if coroutine.cr_suspended:
+ return CORO_SUSPENDED
  if coroutine.cr_frame is None:
  return CORO_CLOSED
- if coroutine.cr_frame.f_lasti == -1:
- return CORO_CREATED
- return CORO_SUSPENDED
+ return CORO_CREATED
 
 
 def getcoroutinelocals(coroutine):

Lib/opcode.py

@@ -94,6 +94,7 @@ def jabs_op(name, op):
 
 def_op('GET_AWAITABLE', 73)
 def_op('LOAD_ASSERTION_ERROR', 74)
+def_op('RETURN_GENERATOR', 75)
 
 def_op('LIST_TO_TUPLE', 82)
 def_op('RETURN_VALUE', 83)
@@ -155,7 +156,7 @@ def jabs_op(name, op):
 
 def_op('MAKE_FUNCTION', 132) # Flags
 def_op('BUILD_SLICE', 133) # Number of items
-
+jabs_op('JUMP_NO_INTERRUPT', 134) # Target byte offset from beginning of code
 def_op('MAKE_CELL', 135)
 hasfree.append(135)
 def_op('LOAD_CLOSURE', 136)

Lib/test/test_compile.py

@@ -954,7 +954,7 @@ def return_genexp():
  x
  in
  y)
- genexp_lines = [None, 1, 3, 1]
+ genexp_lines = [1, 3, 1]
 
  genexp_code = return_genexp.__code__.co_consts[1]
  code_lines = [ None if line is None else line-return_genexp.__code__.co_firstlineno
@@ -967,7 +967,7 @@ async def test(aseq):
  async for i in aseq:
  body
 
- expected_lines = [None, 0, 1, 2, 1]
+ expected_lines = [0, 1, 2, 1]
  code_lines = [ None if line is None else line-test.__code__.co_firstlineno
  for (_, _, line) in test.__code__.co_lines() ]
  self.assertEqual(expected_lines, code_lines)

Lib/test/test_generators.py

@@ -897,7 +897,7 @@ def b():
 >>> type(i)
 <class 'generator'>
 >>> [s for s in dir(i) if not s.startswith('_')]
-['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_yieldfrom', 'send', 'throw']
+['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_suspended', 'gi_yieldfrom', 'send', 'throw']
 >>> from test.support import HAVE_DOCSTRINGS
 >>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).')
 Implement next(self).

Lib/test/test_sys.py

@@ -1386,7 +1386,7 @@ class C(object): pass
  def func():
  return sys._getframe()
  x = func()
- check(x, size('3Pi3c8P2ic?P'))
+ check(x, size('3Pi3c7P2ic??P'))
  # function
  def func(): pass
  check(func, size('14Pi'))
@@ -1403,7 +1403,7 @@ def bar(cls):
  check(bar, size('PP'))
  # generator
  def get_gen(): yield 1
- check(get_gen(), size('P2P4P4c8P2ic?P'))
+ check(get_gen(), size('P2P4P4c7P2ic??P'))
  # iterator
  check(iter('abc'), size('lP'))
  # callable-iterator

Misc/NEWS.d/next/Core and Builtins/2022-01-17-12-57-27.bpo-46409.HouS6m.rst

@@ -0,0 +1,6 @@
+Add new ``RETURN_GENERATOR`` bytecode to make generators.
+Simplifies calling Python functions in the VM, as they no
+longer any need to special case generator functions.
+
+Also add ``JUMP_NO_INTERRUPT`` bytecode that acts like
+``JUMP_ABSOLUTE``, but does not check for interrupts.

Objects/frameobject.c

@@ -242,6 +242,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
  break;
  }
  case JUMP_ABSOLUTE:
+ case JUMP_NO_INTERRUPT:
  j = get_arg(code, i);
  assert(j < len);
  if (stacks[j] == UNINITIALIZED && j < i){
@@ -625,7 +626,7 @@ frame_dealloc(PyFrameObject *f)
 {
  /* It is the responsibility of the owning generator/coroutine
  * to have cleared the generator pointer */
- assert(f->f_frame->generator == NULL);
+ assert(!f->f_frame->is_generator);
 
  if (_PyObject_GC_IS_TRACKED(f)){
  _PyObject_GC_UNTRACK(f);
@@ -698,8 +699,11 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
  "cannot clear an executing frame");
  return NULL;
  }
- if (f->f_frame->generator){
- _PyGen_Finalize(f->f_frame->generator);
+ if (f->f_frame->is_generator){
+ assert(!f->f_owns_frame);
+ size_t offset_in_gen = offsetof(PyGenObject, gi_iframe);
+ PyObject *gen = (PyObject *)(((char *)f->f_frame) - offset_in_gen);
+ _PyGen_Finalize(gen);
  }
  (void)frame_tp_clear(f);
  Py_RETURN_NONE;