gh-119180: PEP 649 compiler changes

Include/internal/pycore_global_strings.h

@@ -45,6 +45,7 @@ struct _Py_global_strings{
  STRUCT_FOR_STR(dot, ".")
  STRUCT_FOR_STR(dot_locals, ".<locals>")
  STRUCT_FOR_STR(empty, "")
+ STRUCT_FOR_STR(format, ".format")
  STRUCT_FOR_STR(generic_base, ".generic_base")
  STRUCT_FOR_STR(json_decoder, "json.decoder")
  STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
@@ -234,7 +235,6 @@ struct _Py_global_strings{
  STRUCT_FOR_ID(_abstract_)
  STRUCT_FOR_ID(_active)
  STRUCT_FOR_ID(_align_)
- STRUCT_FOR_ID(_annotation)
  STRUCT_FOR_ID(_anonymous_)
  STRUCT_FOR_ID(_argtypes_)
  STRUCT_FOR_ID(_as_parameter_)

Include/internal/pycore_opcode_utils.h

@@ -57,6 +57,7 @@ extern "C"{
 #define MAKE_FUNCTION_KWDEFAULTS 0x02
 #define MAKE_FUNCTION_ANNOTATIONS 0x04
 #define MAKE_FUNCTION_CLOSURE 0x08
+#define MAKE_FUNCTION_ANNOTATE 0x10
 
 /* Values used as the oparg for LOAD_COMMON_CONSTANT */
 #define CONSTANT_ASSERTIONERROR 0

Include/internal/pycore_symtable.h

@@ -12,8 +12,9 @@ struct _mod; // Type defined in pycore_ast.h
 
 typedef enum _block_type{
  FunctionBlock, ClassBlock, ModuleBlock,
- // Used for annotations if 'from __future__ import annotations' is active.
- // Annotation blocks cannot bind names and are not evaluated.
+ // Used for annotations. If 'from __future__ import annotations' is active,
+ // annotation blocks cannot bind names and are not evaluated. Otherwise, they
+ // are lazily evaluated (see PEP 649).
  AnnotationBlock,
  // Used for generics and type aliases. These work mostly like functions
  // (see PEP 695 for details). The three different blocks function identically;
@@ -89,6 +90,7 @@ typedef struct _symtable_entry{
  including free refs to globals */
  unsigned ste_generator : 1; /* true if namespace is a generator */
  unsigned ste_coroutine : 1; /* true if namespace is a coroutine */
+ unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */
  _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */
  unsigned ste_varargs : 1; /* true if block has varargs */
  unsigned ste_varkeywords : 1; /* true if block has varkeywords */
@@ -110,6 +112,7 @@ typedef struct _symtable_entry{
  int ste_end_col_offset; /* end offset of first line of block */
  int ste_opt_lineno; /* lineno of last exec or import * */
  int ste_opt_col_offset; /* offset of last exec or import * */
+ struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */
  struct symtable *ste_table;
 } PySTEntryObject;
 
@@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build(
  PyObject *filename,
  _PyFutureFeatures *future);
 extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);
+extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **);
 
 extern void _PySymtable_Free(struct symtable *);
 

Lib/inspect.py

@@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False):
  """
  if isinstance(obj, type):
  # class
- obj_dict = getattr(obj, '__dict__', None)
- if obj_dict and hasattr(obj_dict, 'get'):
- ann = obj_dict.get('__annotations__', None)
- if isinstance(ann, types.GetSetDescriptorType):
- ann = None
- else:
- ann = None
+ ann = obj.__annotations__
 
  obj_globals = None
  module_name = getattr(obj, '__module__', None)

Lib/symtable.py

@@ -222,6 +222,8 @@ def get_methods(self):
  if self.__methods is None:
  d ={}
  for st in self._table.children:
+ if st.type == _symtable.TYPE_ANNOTATION:
+ continue
  d[st.name] = 1
  self.__methods = tuple(d)
  return self.__methods

Lib/test/test_dis.py

@@ -352,32 +352,21 @@ def wrap_func_w_kwargs():
 dis_annot_stmt_str = """\
  0 RESUME 0
 
- 2 SETUP_ANNOTATIONS
- LOAD_CONST 0 (1)
+ 2 LOAD_CONST 0 (1)
  STORE_NAME 0 (x)
- LOAD_NAME 1 (int)
- LOAD_NAME 2 (__annotations__)
- LOAD_CONST 1 ('x')
- STORE_SUBSCR
-
- 3 LOAD_NAME 3 (fun)
- PUSH_NULL
- LOAD_CONST 0 (1)
- CALL 1
- LOAD_NAME 2 (__annotations__)
- LOAD_CONST 2 ('y')
- STORE_SUBSCR
 
  4 LOAD_CONST 0 (1)
- LOAD_NAME 4 (lst)
- LOAD_NAME 3 (fun)
+ LOAD_NAME 1 (lst)
+ LOAD_NAME 2 (fun)
  PUSH_NULL
- LOAD_CONST 3 (0)
+ LOAD_CONST 1 (0)
  CALL 1
  STORE_SUBSCR
- LOAD_NAME 1 (int)
- POP_TOP
- RETURN_CONST 4 (None)
+
+ 2 LOAD_CONST 2 (<code object __annotate__ at 0x..., file "<dis>", line 2>)
+ MAKE_FUNCTION
+ STORE_NAME 3 (__annotate__)
+ RETURN_CONST 3 (None)
 """
 
 compound_stmt_str = """\

Lib/test/test_grammar.py

@@ -306,16 +306,6 @@ def test_eof_error(self):
 
 var_annot_global: int # a global annotated is necessary for test_var_annot
 
-# custom namespace for testing __annotations__
-
-class CNS:
- def __init__(self):
- self._dct ={}
- def __setitem__(self, item, value):
- self._dct[item.lower()] = value
- def __getitem__(self, item):
- return self._dct[item]
-
 
 class GrammarTests(unittest.TestCase):
 
@@ -446,22 +436,12 @@ class F(C, A):
  self.assertEqual(E.__annotations__,{})
  self.assertEqual(F.__annotations__,{})
 
-
- def test_var_annot_metaclass_semantics(self):
- class CMeta(type):
- @classmethod
- def __prepare__(metacls, name, bases, **kwds):
- return{'__annotations__': CNS()}
- class CC(metaclass=CMeta):
- XX: 'ANNOT'
- self.assertEqual(CC.__annotations__['xx'], 'ANNOT')
-
  def test_var_annot_module_semantics(self):
  self.assertEqual(test.__annotations__,{})
  self.assertEqual(ann_module.__annotations__,
-{1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
+{'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
  self.assertEqual(ann_module.M.__annotations__,
-{'123': 123, 'o': type})
+{'o': type})
  self.assertEqual(ann_module2.__annotations__,{})
 
  def test_var_annot_in_module(self):
@@ -476,51 +456,12 @@ def test_var_annot_in_module(self):
  ann_module3.D_bad_ann(5)
 
  def test_var_annot_simple_exec(self):
- gns ={}; lns={}
+ gns ={}; lns={}
  exec("'docstring'\n"
- "__annotations__[1] = 2\n"
  "x: int = 5\n", gns, lns)
- self.assertEqual(lns["__annotations__"],{1: 2, 'x': int})
- with self.assertRaises(KeyError):
- gns['__annotations__']
-
- def test_var_annot_custom_maps(self):
- # tests with custom locals() and __annotations__
- ns ={'__annotations__': CNS()}
- exec('X: int; Z: str = "Z" (w): complex = 1j', ns)
- self.assertEqual(ns['__annotations__']['x'], int)
- self.assertEqual(ns['__annotations__']['z'], str)
+ self.assertEqual(lns["__annotate__"](1),{'x': int})
  with self.assertRaises(KeyError):
- ns['__annotations__']['w']
- nonloc_ns ={}
- class CNS2:
- def __init__(self):
- self._dct ={}
- def __setitem__(self, item, value):
- nonlocal nonloc_ns
- self._dct[item] = value
- nonloc_ns[item] = value
- def __getitem__(self, item):
- return self._dct[item]
- exec('x: int = 1',{}, CNS2())
- self.assertEqual(nonloc_ns['__annotations__']['x'], int)
-
- def test_var_annot_refleak(self):
- # complex case: custom locals plus custom __annotations__
- # this was causing refleak
- cns = CNS()
- nonloc_ns ={'__annotations__': cns}
- class CNS2:
- def __init__(self):
- self._dct ={'__annotations__': cns}
- def __setitem__(self, item, value):
- nonlocal nonloc_ns
- self._dct[item] = value
- nonloc_ns[item] = value
- def __getitem__(self, item):
- return self._dct[item]
- exec('X: str',{}, CNS2())
- self.assertEqual(nonloc_ns['__annotations__']['x'], str)
+ gns['__annotate__']
 
  def test_var_annot_rhs(self):
  ns ={}

Lib/test/test_module/__init__.py

@@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self):
  ann_module4 = import_helper.import_fresh_module(
  'test.typinganndata.ann_module4',
  )
+ self.assertFalse("__annotations__" in ann_module4.__dict__)
+ self.assertEqual(ann_module4.__annotations__,{"a": int, "b": str})
  self.assertTrue("__annotations__" in ann_module4.__dict__)
  del ann_module4.__annotations__
  self.assertFalse("__annotations__" in ann_module4.__dict__)

Lib/test/test_opcodes.py

@@ -39,16 +39,19 @@ class C: pass
  def test_use_existing_annotations(self):
  ns ={'__annotations__':{1: 2}}
  exec('x: int', ns)
- self.assertEqual(ns['__annotations__'],{'x': int, 1: 2})
+ self.assertEqual(ns['__annotations__'],{1: 2})
 
  def test_do_not_recreate_annotations(self):
  # Don't rely on the existence of the '__annotations__' global.
  with support.swap_item(globals(), '__annotations__',{}):
-del globals()['__annotations__']
+ globals().pop('__annotations__', None)
  class C:
- del __annotations__
- with self.assertRaises(NameError):
- x: int
+ try:
+ del __annotations__
+ except NameError:
+ pass
+ x: int
+ self.assertEqual(C.__annotations__,{"x": int})
 
  def test_raise_class_exceptions(self):
 

Lib/test/test_positional_only_arg.py

@@ -2,6 +2,7 @@
 
 import dis
 import pickle
+import types
 import unittest
 
 from test.support import check_syntax_error
@@ -440,7 +441,9 @@ def f(x: not (int is int), /): ...
  # without constant folding we end up with
  # COMPARE_OP(is), IS_OP (0)
  # with constant folding we should expect a IS_OP (1)
- codes = [(i.opname, i.argval) for i in dis.get_instructions(g)]
+ code_obj = next(const for const in g.__code__.co_consts
+ if isinstance(const, types.CodeType) and const.co_name == "__annotate__")
+ codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)]
  self.assertNotIn(('UNARY_NOT', None), codes)
  self.assertIn(('IS_OP', 1), codes)
 

Lib/test/test_pyclbr.py

@@ -109,6 +109,8 @@ def ismethod(oclass, obj, name):
 
  actualMethods = []
  for m in py_item.__dict__.keys():
+ if m == "__annotate__":
+ continue
  if ismethod(py_item, getattr(py_item, m), m):
  actualMethods.append(m)
  foundMethods = []

Lib/test/test_pydoc/test_pydoc.py

@@ -77,6 +77,11 @@ class A(builtins.object)
  | __weakref__%s
 
  class B(builtins.object)
+ | Methods defined here:
+ |
+ | __annotate__(...)
+ |
+ | ----------------------------------------------------------------------
  | Data descriptors defined here:
  |
  | __dict__%s
@@ -87,8 +92,6 @@ class B(builtins.object)
  | Data and other attributes defined here:
  |
  | NO_MEANING = 'eggs'
- |
- | __annotations__ ={'NO_MEANING': <class 'str'>}
 
  class C(builtins.object)
  | Methods defined here:
@@ -176,6 +179,9 @@ class A(builtins.object)
  list of weak references to the object
 
 class B(builtins.object)
+ Methods defined here:
+ __annotate__(...)
+ ----------------------------------------------------------------------
  Data descriptors defined here:
  __dict__
  dictionary for instance variables
@@ -184,7 +190,6 @@ class B(builtins.object)
  ----------------------------------------------------------------------
  Data and other attributes defined here:
  NO_MEANING = 'eggs'
- __annotations__ ={'NO_MEANING': <class 'str'>}
 
 
 class C(builtins.object)

Lib/test/test_pyrepl/test_interact.py

@@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self):
 
  def test_no_active_future(self):
  console = InteractiveColoredConsole()
- source = "x: int = 1; print(__annotations__)"
+ source = "x: int = 1; print(__annotate__(1))"
  f = io.StringIO()
  with contextlib.redirect_stdout(f):
  result = console.runsource(source)

Lib/test/test_symtable.py

@@ -205,12 +205,14 @@ def test_assigned(self):
 
  def test_annotated(self):
  st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
- st2 = st1.get_children()[0]
+ st2 = st1.get_children()[1]
+ self.assertEqual(st2.get_type(), "function")
  self.assertTrue(st2.lookup('x').is_local())
  self.assertTrue(st2.lookup('x').is_annotated())
  self.assertFalse(st2.lookup('x').is_global())
  st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
- st4 = st3.get_children()[0]
+ st4 = st3.get_children()[1]
+ self.assertEqual(st4.get_type(), "function")
  self.assertTrue(st4.lookup('x').is_local())
  self.assertFalse(st4.lookup('x').is_annotated())