Skip to content

bpo-43224: Implement PEP 646 grammar changes#31018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JelleZijlstra merged 17 commits into from
Mar 26, 2022
Merged

bpo-43224: Implement PEP 646 grammar changes #31018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7ecb7ba
bpo-43224: Implement PEP 646 grammar changes
mrahtz Jan 30, 2022
c0531ec
Add tests for del
mrahtz Jan 30, 2022
479c175
Tweak news to better fit in with other PRs
mrahtz Jan 30, 2022
091ceeb
Move error cases into test_syntax
mrahtz Feb 13, 2022
f1c9c77
Add an extra failure test
mrahtz Feb 14, 2022
c8d2534
Add more tests combining slicing with starring
mrahtz Mar 17, 2022
5df1e97
Fix ast representation of slices combined with starring
mrahtz Mar 17, 2022
677a326
Fix broken test in test_unparse
mrahtz Mar 17, 2022
1543785
Add a check_src_roundtrip test for "a[1:2, *a]"
mrahtz Mar 17, 2022
e1ab6ea
fix whitespace (make patchcheck)
JelleZijlstra Mar 23, 2022
18e7fa7
Merge remote-tracking branch 'upstream/main' into pep646-grammar
JelleZijlstra Mar 23, 2022
72dd8b9
Add tests for AST changes
mrahtz Mar 24, 2022
80bb5fa
Fix C unpack
mrahtz Mar 24, 2022
fbfd5b1
Fix typing.get_type_hints on variadic annotation
mrahtz Mar 24, 2022
816098c
Fix typo
mrahtz Mar 24, 2022
8962d26
Remove debug print
mrahtz Mar 25, 2022
973e9cc
Fix potential shadowing in ForwardRef
mrahtz Mar 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Grammar/python.gram
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -309,6 +309,8 @@ star_etc[StarEtc*]:
| invalid_star_etc
| '*' a=param_no_default b=param_maybe_default* c=[kwds]{
_PyPegen_star_etc(p, a, b, c) }
| '*' a=param_no_default_star_annotation b=param_maybe_default* c=[kwds]{
_PyPegen_star_etc(p, a, b, c) }
| '*' ',' b=param_maybe_default+ c=[kwds]{
_PyPegen_star_etc(p, NULL, b, c) }
| a=kwds{_PyPegen_star_etc(p, NULL, NULL, a) }
Expand All@@ -333,14 +335,19 @@ kwds[arg_ty]:
param_no_default[arg_ty]:
| a=param ',' tc=TYPE_COMMENT?{_PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param tc=TYPE_COMMENT? &')'{_PyPegen_add_type_comment_to_arg(p, a, tc) }
param_no_default_star_annotation[arg_ty]:
| a=param_star_annotation ',' tc=TYPE_COMMENT?{_PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param_star_annotation tc=TYPE_COMMENT? &')'{_PyPegen_add_type_comment_to_arg(p, a, tc) }
param_with_default[NameDefaultPair*]:
| a=param c=default ',' tc=TYPE_COMMENT?{_PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default tc=TYPE_COMMENT? &')'{_PyPegen_name_default_pair(p, a, c, tc) }
param_maybe_default[NameDefaultPair*]:
| a=param c=default? ',' tc=TYPE_COMMENT?{_PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default? tc=TYPE_COMMENT? &')'{_PyPegen_name_default_pair(p, a, c, tc) }
param[arg_ty]: a=NAME b=annotation?{_PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
param_star_annotation[arg_ty]: a=NAME b=star_annotation{_PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
annotation[expr_ty]: ':' a=expression{a }
star_annotation[expr_ty]: ':' a=star_expression{a }
default[expr_ty]: '=' a=expression{a } | invalid_default

# If statement
Expand DownExpand Up@@ -782,7 +789,7 @@ primary[expr_ty]:

slices[expr_ty]:
| a=slice !','{a }
| a[asdl_expr_seq*]=','.slice+ [',']{_PyAST_Tuple(a, Load, EXTRA) }
| a[asdl_expr_seq*]=','.(slice | starred_expression)+ [',']{_PyAST_Tuple(a, Load, EXTRA) }

slice[expr_ty]:
| a=[expression] ':' b=[expression] c=[':' d=[expression]{d }]{_PyAST_Slice(a, b, c, EXTRA) }
Expand Down
9 changes: 3 additions & 6 deletions Lib/ast.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1476,20 +1476,17 @@ def visit_Call(self, node):
self.traverse(e)

def visit_Subscript(self, node):
def is_simple_tuple(slice_value):
# when unparsing a non-empty tuple, the parentheses can be safely
# omitted if there aren't any elements that explicitly requires
# parentheses (such as starred expressions).
def is_non_empty_tuple(slice_value):
return (
isinstance(slice_value, Tuple)
and slice_value.elts
and not any(isinstance(elt, Starred) for elt in slice_value.elts)
)

self.set_precedence(_Precedence.ATOM, node.value)
self.traverse(node.value)
with self.delimit("[", "]"):
if is_simple_tuple(node.slice):
if is_non_empty_tuple(node.slice):
# parentheses can be omitted if the tuple isn't empty
self.items_view(self.traverse, node.slice.elts)
else:
self.traverse(node.slice)
Expand Down
25 changes: 24 additions & 1 deletion Lib/test/test_ast.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,7 +13,7 @@
from test import support

def to_tuple(t):
if t is None or isinstance(t, (str, int, complex)):
if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis:
return t
elif isinstance(t, list):
return [to_tuple(e) for e in t]
Expand DownExpand Up@@ -46,10 +46,20 @@ def to_tuple(t):
"def f(a=0): pass",
# FunctionDef with varargs
"def f(*args): pass",
# FunctionDef with varargs as TypeVarTuple
"def f(*args: *Ts): pass",
# FunctionDef with varargs as unpacked Tuple
"def f(*args: *tuple[int, ...]): pass",
# FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple
"def f(*args: *tuple[int, *Ts]): pass",
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args and docstring
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
# FunctionDef with type annotation on return involving unpacking
"def f() -> tuple[*Ts]: pass",
"def f() -> tuple[int, *Ts]: pass",
"def f() -> tuple[int, *tuple[int, ...]]: pass",
# ClassDef
"class C:pass",
# ClassDef with docstring
Expand All@@ -65,6 +75,10 @@ def to_tuple(t):
"a,b = c",
"(a,b) = c",
"[a,b] = c",
# AnnAssign with unpacked types
"x: tuple[*Ts]",
"x: tuple[int, *Ts]",
"x: tuple[int, *tuple[str, ...]]",
# AugAssign
"v += 1",
# For
Expand DownExpand Up@@ -2315,8 +2329,14 @@ def main():
('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [])], []),
Expand All@@ -2326,6 +2346,9 @@ def main():
('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []),
('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []),
('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []),
Expand Down
41 changes: 38 additions & 3 deletions Lib/test/test_future.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ def _exec_future(self, code):
scope ={}
exec(
"from __future__ import annotations\n"
+ code, {}, scope
Copy link
ContributorAuthor

@mrahtzmrahtzMar 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed this because if both globals and locals are passed to exec, it treats the code as it if were executed in a class definition, which makes the new test I've added harder than it needs to be (we can't access c so easily). Removing this doesn't seem to break any of the rest of the tests, so I hope this is fine?

+ code, scope
)
return scope

Expand DownExpand Up@@ -287,10 +287,11 @@ def test_annotations(self):
eq("list[str]")
eq("dict[str, int]")
eq("set[str,]")
eq("tuple[()]")
eq("tuple[str, ...]")
eq("tuple[(str, *types)]")
eq("tuple[str, *types]")
eq("tuple[str, int, (str, int)]")
eq("tuple[(*int, str, str, (str, int))]")
eq("tuple[*int, str, str, (str, int)]")
eq("tuple[str, int, float, dict[str, int]]")
eq("slice[0]")
eq("slice[0:1]")
Expand All@@ -305,6 +306,21 @@ def test_annotations(self):
eq("slice[1:2, 1]")
eq("slice[1:2, 2, 3]")
eq("slice[()]")
# Note that `slice[*Ts]`, `slice[*Ts,]`, and `slice[(*Ts,)]` all have
# the same AST, but only `slice[*Ts,]` passes this test, because that's
# what the unparser produces.
eq("slice[*Ts,]")
eq("slice[1, *Ts]")
eq("slice[*Ts, 2]")
eq("slice[1, *Ts, 2]")
eq("slice[*Ts, *Ts]")
eq("slice[1, *Ts, *Ts]")
eq("slice[*Ts, 1, *Ts]")
eq("slice[*Ts, *Ts, 1]")
eq("slice[1, *Ts, *Ts, 2]")
eq("slice[1:2, *Ts]")
eq("slice[*Ts, 1:2]")
eq("slice[1:2, *Ts, 3:4]")
eq("slice[a, b:c, d:e:f]")
eq("slice[(x for x in a)]")
eq('str or None if sys.version_info[0] > (3,) else str or bytes or None')
Expand DownExpand Up@@ -403,6 +419,25 @@ def foo():
def bar(arg: (yield)): pass
"""))

def test_get_type_hints_on_func_with_variadic_arg(self):
# `typing.get_type_hints` might break on a function with a variadic
# annotation (e.g. `f(*args: *Ts)`) if `from __future__ import
# annotations`, because it could try to evaluate `*Ts` as an expression,
# which on its own isn't value syntax.
namespace = self._exec_future(dedent("""\
class StarredC: pass
class C:
def __iter__(self):
yield StarredC()
c = C()
def f(*args: *c): pass
import typing
hints = typing.get_type_hints(f)
"""))

hints = namespace.pop('hints')
self.assertIsInstance(hints['args'], namespace['StarredC'])


if __name__ == "__main__":
unittest.main()
Loading