python · ericsnowcurrently · Apr 22, 2023 · Apr 11, 2020 · Apr 11, 2020 · Apr 11, 2020
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
@@ -670,6 +670,13 @@ always available.
  .. versionadded:: 3.4
 
 
+.. function:: getunicodeinternedsize()
+
+ Return the number of unicode objects that have been interned.
+
+ .. versionadded:: 3.12
+
+
 .. function:: getandroidapilevel()
 
  Return the build time API version of Android as an integer.

@@ -1129,6 +1129,24 @@ New Features
  to replace the legacy-api :c:func:`!PyErr_Display`. (Contributed by
  Irit Katriel in :gh:`102755`).
 
+* :pep:`683`: Introduced Immortal Objects to Python which allows objects
+ to bypass reference counts and introduced changes to the C-API:
+
+ - ``_Py_IMMORTAL_REFCNT``: The reference count that defines an object
+ as immortal.
+ - ``_Py_IsImmortal`` Checks if an object has the immortal reference count.
+ - ``PyObject_HEAD_INIT`` This will now initialize reference count to
+ ``_Py_IMMORTAL_REFCNT`` when used with ``Py_BUILD_CORE``.
+ - ``SSTATE_INTERNED_IMMORTAL`` An identifier for interned unicode objects
+ that are immortal.
+ - ``SSTATE_INTERNED_IMMORTAL_STATIC`` An identifier for interned unicode
+ objects that are immortal and static
+ - ``sys.getunicodeinternedsize`` This returns the total number of unicode
+ objects that have been interned. This is now needed for refleak.py to
+ correctly track reference counts and allocated blocks
+
+ (Contributed by Eddie Elizondo in :gh:`84436`.)
+
 Porting to Python 3.12
 ----------------------
 
@@ -1293,8 +1311,7 @@ Removed
  * :c:func:`!PyUnicode_GetSize`
  * :c:func:`!PyUnicode_GET_DATA_SIZE`
 
-* Remove the ``PyUnicode_InternImmortal()`` function and the
- ``SSTATE_INTERNED_IMMORTAL`` macro.
+* Remove the ``PyUnicode_InternImmortal()`` function macro.
  (Contributed by Victor Stinner in :gh:`85858`.)
 
 * Remove ``Jython`` compatibility hacks from several stdlib modules and tests.

diff --git a/Include/boolobject.h b/Include/boolobject.h
@@ -11,8 +11,7 @@ PyAPI_DATA(PyTypeObject) PyBool_Type;
 
 #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type)
 
-/* Py_False and Py_True are the only two bools in existence.
-Don't forget to apply Py_INCREF() when returning either!!! */
+/* Py_False and Py_True are the only two bools in existence. */
 
 /* Don't use these directly */
 PyAPI_DATA(PyLongObject) _Py_FalseStruct;
@@ -31,8 +30,8 @@ PyAPI_FUNC(int) Py_IsFalse(PyObject *x);
 #define Py_IsFalse(x) Py_Is((x), Py_False)
 
 /* Macros for returning Py_True or Py_False, respectively */
-#define Py_RETURN_TRUE return Py_NewRef(Py_True)
-#define Py_RETURN_FALSE return Py_NewRef(Py_False)
+#define Py_RETURN_TRUE return Py_True
+#define Py_RETURN_FALSE return Py_False
 
 /* Function to return a bool from a C long */
 PyAPI_FUNC(PyObject *) PyBool_FromLong(long);

diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
@@ -98,9 +98,16 @@ typedef struct{
  Py_ssize_t length; /* Number of code points in the string */
  Py_hash_t hash; /* Hash value; -1 if not set */
  struct{
- /* If interned is set, the two references from the
- dictionary to this object are *not* counted in ob_refcnt. */
- unsigned int interned:1;
+ /* If interned is non-zero, the two references from the
+ dictionary to this object are *not* counted in ob_refcnt.
+ The possible values here are:
+ 0: Not Interned
+ 1: Interned
+ 2: Interned and Immortal
+ 3: Interned, Immortal, and Static
+ This categorization allows the runtime to determine the right
+ cleanup mechanism at runtime shutdown. */
+ unsigned int interned:2;
  /* Character size:
 
  - PyUnicode_1BYTE_KIND (1):
@@ -135,7 +142,7 @@ typedef struct{
  unsigned int ascii:1;
  /* Padding to ensure that PyUnicode_DATA() is always aligned to
  4 bytes (see issue #19537 on m68k). */
- unsigned int :26;
+ unsigned int :25;
  } state;
 } PyASCIIObject;
 
@@ -183,6 +190,8 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
 /* Interning state. */
 #define SSTATE_NOT_INTERNED 0
 #define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+#define SSTATE_INTERNED_IMMORTAL_STATIC 3
 
 /* Use only if you know it's a string */
 static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op){

diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
@@ -245,7 +245,7 @@ _PyLong_FlipSign(PyLongObject *op){
 
 #define _PyLong_DIGIT_INIT(val) \
 {\
- .ob_base = _PyObject_IMMORTAL_INIT(&PyLong_Type), \
+ .ob_base = _PyObject_HEAD_INIT(&PyLong_Type) \
  .long_value ={\
  .lv_tag = TAG_FROM_SIGN_AND_SIZE( \
  (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \

diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -14,21 +14,25 @@ extern "C"{
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
 #include "pycore_runtime.h" // _PyRuntime
 
-/* This value provides *effective* immortality, meaning the object should never
- be deallocated (until runtime finalization). See PEP 683 for more details about
- immortality, as well as a proposed mechanism for proper immortality. */
-#define _PyObject_IMMORTAL_REFCNT 999999999
-
-#define _PyObject_IMMORTAL_INIT(type) \
-{\
- .ob_refcnt = _PyObject_IMMORTAL_REFCNT, \
- .ob_type = (type), \
- }
-#define _PyVarObject_IMMORTAL_INIT(type, size) \
-{\
- .ob_base = _PyObject_IMMORTAL_INIT(type), \
- .ob_size = size, \
- }
+/* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid
+ designated initializer conflicts in C++20. If we use the deinition in
+ object.h, we will be mixing designated and non-designated initializers in
+ pycore objects which is forbiddent in C++20. However, if we then use
+ designated initializers in object.h then Extensions without designated break.
+ Furthermore, we can't use designated initializers in Extensions since these
+ are not supported pre-C++20. Thus, keeping an internal copy here is the most
+ backwards compatible solution */
+#define _PyObject_HEAD_INIT(type) \
+{\
+ _PyObject_EXTRA_INIT \
+ .ob_refcnt = _Py_IMMORTAL_REFCNT, \
+ .ob_type = (type) \
+ },
+#define _PyVarObject_HEAD_INIT(type, size) \
+{\
+ .ob_base = _PyObject_HEAD_INIT(type) \
+ .ob_size = size \
+ },
 
 PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
  const char *func,
@@ -61,9 +65,20 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n)
 }
 #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n)
 
+static inline void _Py_SetImmortal(PyObject *op)
+{
+ if (op){
+ op->ob_refcnt = _Py_IMMORTAL_REFCNT;
+ }
+}
+#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op))
+
 static inline void
 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 {
+ if (_Py_IsImmortal(op)){
+ return;
+ }
  _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
  _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
@@ -82,6 +97,9 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 static inline void
 _Py_DECREF_NO_DEALLOC(PyObject *op)
 {
+ if (_Py_IsImmortal(op)){
+ return;
+ }
  _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
  _Py_DEC_REFTOTAL(_PyInterpreterState_GET());

@@ -76,13 +76,13 @@ extern PyTypeObject _PyExc_MemoryError;
  .latin1 = _Py_str_latin1_INIT, \
  }, \
  .tuple_empty ={\
- .ob_base = _PyVarObject_IMMORTAL_INIT(&PyTuple_Type, 0) \
+ .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0) \
  }, \
  .hamt_bitmap_node_empty ={\
- .ob_base = _PyVarObject_IMMORTAL_INIT(&_PyHamt_BitmapNode_Type, 0) \
+ .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0) \
  }, \
  .context_token_missing ={\
- .ob_base = _PyObject_IMMORTAL_INIT(&_PyContextTokenMissing_Type), \
+ .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type) \
  }, \
  }, \
  }, \
@@ -116,11 +116,11 @@ extern PyTypeObject _PyExc_MemoryError;
  .singletons ={\
  ._not_used = 1, \
  .hamt_empty ={\
- .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \
+ .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type) \
  .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \
  }, \
  .last_resort_memory_error ={\
-_PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \
+_PyObject_HEAD_INIT(&_PyExc_MemoryError) \
  }, \
  }, \
  }, \
@@ -138,7 +138,7 @@ extern PyTypeObject _PyExc_MemoryError;
 
 #define _PyBytes_SIMPLE_INIT(CH, LEN) \
 {\
-_PyVarObject_IMMORTAL_INIT(&PyBytes_Type, (LEN)), \
+_PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)) \
  .ob_shash = -1, \
  .ob_sval ={(CH) }, \
  }
@@ -149,7 +149,7 @@ extern PyTypeObject _PyExc_MemoryError;
 
 #define _PyUnicode_ASCII_BASE_INIT(LITERAL, ASCII) \
 {\
- .ob_base = _PyObject_IMMORTAL_INIT(&PyUnicode_Type), \
+ .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type) \
  .length = sizeof(LITERAL) - 1, \
  .hash = -1, \
  .state ={\

diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h
@@ -12,6 +12,7 @@ extern "C"{
 #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
 
 void _PyUnicode_ExactDealloc(PyObject *op);
+Py_ssize_t _PyUnicode_InternedSize(void);
 
 /* runtime lifecycle */