gh-130704: Strength reduce LOAD_FAST{_LOAD_FAST}

Doc/library/dis.rst

@@ -76,7 +76,7 @@ the following command can be used to display the disassembly of
  2 RESUME 0
  <BLANKLINE>
  3 LOAD_GLOBAL 1 (len + NULL)
-LOAD_FAST  0 (alist)
+LOAD_FAST_BORROW 0 (alist)
  CALL 1
  RETURN_VALUE
 
@@ -215,7 +215,7 @@ Example:
  ...
  RESUME
  LOAD_GLOBAL
-LOAD_FAST
+LOAD_FAST_BORROW
  CALL
  RETURN_VALUE
 
@@ -1402,13 +1402,28 @@ iterations of the loop.
  This opcode is now only used in situations where the local variable is
  guaranteed to be initialized. It cannot raise :exc:`UnboundLocalError`.
 
+.. opcode:: LOAD_FAST_BORROW (var_num)
+
+ Pushes a borrowed reference to the local ``co_varnames[var_num]`` onto the
+ stack.
+
+ .. versionadded:: 3.14
+
 .. opcode:: LOAD_FAST_LOAD_FAST (var_nums)
 
  Pushes references to ``co_varnames[var_nums >> 4]`` and
  ``co_varnames[var_nums & 15]`` onto the stack.
 
  .. versionadded:: 3.13
 
+
+.. opcode:: LOAD_FAST_BORROW_LOAD_FAST_BORROW (var_nums)
+
+ Pushes borrowed references to ``co_varnames[var_nums >> 4]`` and
+ ``co_varnames[var_nums & 15]`` onto the stack.
+
+ .. versionadded:: 3.14
+
 .. opcode:: LOAD_FAST_CHECK (var_num)
 
  Pushes a reference to the local ``co_varnames[var_num]`` onto the stack,
@@ -2023,4 +2038,3 @@ instructions:
 
  .. deprecated:: 3.13
  All jumps are now relative. This list is empty.
-

Include/internal/pycore_frame.h

@@ -28,6 +28,12 @@ struct _frame{
  PyEval_GetLocals requires a borrowed reference so the actual reference
  is stored here */
  PyObject *f_locals_cache;
+ /* A tuple containing strong references to fast locals that were overwritten
+ * via f_locals. Borrowed references to these locals may exist in frames
+ * closer to the top of the stack. The references in this tuple act as
+ * "support" for the borrowed references, ensuring that they remain valid.
+ */
+ PyObject *f_overwritten_fast_locals;
  /* The frame data, if this frame object owns the frame */
  PyObject *_f_frame_data[1];
 };

Include/internal/pycore_stackref.h

@@ -172,6 +172,12 @@ PyStackRef_MakeHeapSafe(_PyStackRef ref)
  return ref;
 }
 
+static inline _PyStackRef
+PyStackRef_Borrow(_PyStackRef ref)
+{
+ return PyStackRef_DUP(ref)
+}
+
 #define PyStackRef_CLEAR(REF) \
  do{\
  _PyStackRef *_tmp_op_ptr = &(REF); \
@@ -253,6 +259,25 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj)
 }
 # define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
 
+static inline bool
+PyStackRef_IsHeapSafe(_PyStackRef stackref)
+{
+ if (PyStackRef_IsDeferred(stackref)){
+ PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
+ return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj);
+ }
+ return true;
+}
+
+static inline _PyStackRef
+PyStackRef_MakeHeapSafe(_PyStackRef stackref)
+{
+ if (PyStackRef_IsHeapSafe(stackref)){
+ return stackref;
+ }
+ PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
+ return (_PyStackRef){.bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
+}
 
 static inline _PyStackRef
 PyStackRef_FromPyObjectStealMortal(PyObject *obj)
@@ -311,25 +336,16 @@ PyStackRef_DUP(_PyStackRef stackref)
 {
  assert(!PyStackRef_IsNull(stackref));
  if (PyStackRef_IsDeferred(stackref)){
- assert(_Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) ||
- _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref))
- );
  return stackref;
  }
  Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
  return stackref;
 }
 
-static inline bool
-PyStackRef_IsHeapSafe(_PyStackRef ref)
-{
- return true;
-}
-
 static inline _PyStackRef
-PyStackRef_MakeHeapSafe(_PyStackRef ref)
+PyStackRef_Borrow(_PyStackRef stackref)
 {
- return ref;
+ return (_PyStackRef){.bits = stackref.bits | Py_TAG_DEFERRED };
 }
 
 // Convert a possibly deferred reference to a strong reference.
@@ -399,7 +415,6 @@ static inline void PyStackRef_CheckValid(_PyStackRef ref){
  assert(!_Py_IsStaticImmortal(obj));
  break;
  case Py_TAG_REFCNT:
- assert(obj == NULL || _Py_IsImmortal(obj));
  break;
  default:
  assert(0);
@@ -413,21 +428,28 @@ static inline void PyStackRef_CheckValid(_PyStackRef ref){
 #endif
 
 #ifdef _WIN32
-#define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_BITS) == 0)
+#define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_REFCNT) == 0)
 #define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED
+#define PyStackRef_Borrow(REF) (_PyStackRef){.bits = ((REF).bits) | Py_TAG_REFCNT};
 #else
 /* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */
 static inline int
 PyStackRef_RefcountOnObject(_PyStackRef ref)
 {
- return (ref.bits & Py_TAG_BITS) == 0;
+ return (ref.bits & Py_TAG_REFCNT) == 0;
 }
 
 static inline PyObject *
 PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
 {
  return BITS_TO_PTR_MASKED(ref);
 }
+
+static inline _PyStackRef
+PyStackRef_Borrow(_PyStackRef ref)
+{
+ return (_PyStackRef){.bits = ref.bits | Py_TAG_REFCNT };
+}
 #endif
 
 static inline PyObject *