compiler: symbol table framework, plus test machinery

Author: ncw

compile/compile_data_test.go

@@ -1151,7 +1151,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 (<code object <lambda> at 0x7fe734191db0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('<lambda>')\n 6 MAKE_FUNCTION 0\n 9 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 (<code object <lambda> at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('<lambda>')\n 6 MAKE_FUNCTION 0\n 9 RETURN_VALUE\n", nil, ""},
 {"pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -1951,7 +1951,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe7341918a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bccdc8a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
 {"def fn(a): pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -1984,7 +1984,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe73567c6f0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7be1c76f0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
 {"def fn(a,b,c): pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -2017,7 +2017,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe734191db0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
 {"def fn(a,b=1,c=2): pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -2050,7 +2050,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 (1)\n 3 LOAD_CONST 1 (2)\n 6 LOAD_CONST 2 (<code object fn at 0x7fe7341b68a0, file \"<string>\", line 1>)\n 9 LOAD_CONST 3 ('fn')\n 12 MAKE_FUNCTION 2\n 15 STORE_NAME 0 (fn)\n 18 LOAD_CONST 4 (None)\n 21 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 (1)\n 3 LOAD_CONST 1 (2)\n 6 LOAD_CONST 2 (<code object fn at 0x7fd7bcd028a0, file \"<string>\", line 1>)\n 9 LOAD_CONST 3 ('fn')\n 12 MAKE_FUNCTION 2\n 15 STORE_NAME 0 (fn)\n 18 LOAD_CONST 4 (None)\n 21 RETURN_VALUE\n", nil, ""},
 {"def fn(a,*arg,b=1,c=2): pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -2083,7 +2083,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fe7341918a0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fd7bccdc8a0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
 {"def fn(a,*arg,b=1,c=2,**kwargs): pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -2116,7 +2116,7 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fe73567c6f0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fd7be1c76f0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
 {"def fn(a:\"a\",*arg:\"arg\",b:\"b\"=1,c:\"c\"=2,**kwargs:\"kw\") -> \"ret\": pass", "exec", &py.Code{
 Argcount: 0,
 Kwonlyargcount: 0,
@@ -2149,5 +2149,38 @@ var compileTestData = []struct{
 Name: "<module>",
 Firstlineno: 1,
 Lnotab: "",
- }, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 ('a')\n 15 LOAD_CONST 5 ('arg')\n 18 LOAD_CONST 0 ('b')\n 21 LOAD_CONST 2 ('c')\n 24 LOAD_CONST 6 ('kw')\n 27 LOAD_CONST 7 ('ret')\n 30 LOAD_CONST 8 (('a', 'arg', 'b', 'c', 'kwargs', 'return'))\n 33 LOAD_CONST 9 (<code object fn at 0x7fe734191db0, file \"<string>\", line 1>)\n 36 LOAD_CONST 10 ('fn')\n 39 EXTENDED_ARG 7\n 42 MAKE_FUNCTION 459264\n 45 STORE_NAME 0 (fn)\n 48 LOAD_CONST 11 (None)\n 51 RETURN_VALUE\n", nil, ""},
+ }, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 ('a')\n 15 LOAD_CONST 5 ('arg')\n 18 LOAD_CONST 0 ('b')\n 21 LOAD_CONST 2 ('c')\n 24 LOAD_CONST 6 ('kw')\n 27 LOAD_CONST 7 ('ret')\n 30 LOAD_CONST 8 (('a', 'arg', 'b', 'c', 'kwargs', 'return'))\n 33 LOAD_CONST 9 (<code object fn at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 36 LOAD_CONST 10 ('fn')\n 39 EXTENDED_ARG 7\n 42 MAKE_FUNCTION 459264\n 45 STORE_NAME 0 (fn)\n 48 LOAD_CONST 11 (None)\n 51 RETURN_VALUE\n", nil, ""},
+{"def fn(): a+b", "exec", &py.Code{
+Argcount: 0,
+Kwonlyargcount: 0,
+Nlocals: 0,
+Stacksize: 2,
+Flags: 64,
+Code: "\x64\x00\x00\x64\x01\x00\x84\x00\x00\x5a\x00\x00\x64\x02\x00\x53",
+Consts: []py.Object{&py.Code{
+Argcount: 0,
+Kwonlyargcount: 0,
+Nlocals: 0,
+Stacksize: 2,
+Flags: 67,
+Code: "\x74\x00\x00\x74\x01\x00\x17\x01\x64\x00\x00\x53",
+Consts: []py.Object{py.None},
+Names: []string{"a", "b"},
+Varnames: []string{},
+Freevars: []string{},
+Cellvars: []string{},
+Filename: "<string>",
+Name: "fn",
+Firstlineno: 1,
+Lnotab: "",
+ }, py.String("fn"), py.None},
+Names: []string{"fn"},
+Varnames: []string{},
+Freevars: []string{},
+Cellvars: []string{},
+Filename: "<string>",
+Name: "<module>",
+Firstlineno: 1,
+Lnotab: "",
+ }, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bcd028a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
 }

compile/make_compile_test.py

@@ -164,7 +164,7 @@
  ('''def fn(a,*arg,b=1,c=2): pass''', "exec"),
  ('''def fn(a,*arg,b=1,c=2,**kwargs): pass''', "exec"),
  ('''def fn(a:"a",*arg:"arg",b:"b"=1,c:"c"=2,**kwargs:"kw") -> "ret": pass''', "exec"),
-#('''def fn(): a+b''', "exec"),
+ ('''def fn(): a+b''', "exec"),
 
 ]
 

compile/make_symtable_test.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3.4
+"""
+Write symtable_data_test.go
+"""
+
+importsys
+importast
+importsubprocess
+importdis
+fromsymtableimportsymtable
+
+# FIXME test errors too
+
+inp= [
+ ('''1''', "eval"),
+#('''a*b*c''', "eval"),
+#('''def fn(): pass''', "exec"),
+#('''def fn(a,b):\n e=1\n return a*b*c*d*e''', "exec"),
+#('''def fn(a,b):\n def nested(c,d):\n return a*b*c*d*e''', "exec"),
+# ('''def fn(a,b,c): pass''', "exec"),
+# ('''def fn(a,b=1,c=2): pass''', "exec"),
+# ('''def fn(a,*arg,b=1,c=2): pass''', "exec"),
+# ('''def fn(a,*arg,b=1,c=2,**kwargs): pass''', "exec"),
+# ('''def fn(a:"a",*arg:"arg",b:"b"=1,c:"c"=2,**kwargs:"kw") -> "ret": pass''', "exec"),
+# ('''def fn(): a+b''', "exec"),
+]
+
+defdump_bool(b):
+return ("true"ifbelse"false")
+
+defdump_strings(ss):
+return"[]string{"+",".join([ '"%s"'%sforsinss ])+"}"
+
+# Scope numbers to names (from symtable.h)
+SCOPES={
+1: "scopeLocal",
+2: "scopeGlobalExplicit",
+3: "scopeGlobalImplicit",
+4: "scopeFree",
+5: "scopeCell",
+}
+
+#def-use flags to names (from symtable.h)
+DEF_FLAGS= (
+ ("defGlobal", 1), # global stmt
+ ("defLocal", 2), # assignment in code block
+ ("defParam", 2<<1), # formal parameter
+ ("defNonlocal", 2<<2), # nonlocal stmt
+ ("defUse", 2<<3), # name is used
+ ("defFree", 2<<4), # name used but not defined in nested block
+ ("defFreeClass", 2<<5),# free variable from class's method
+ ("defImport", 2<<6), # assignment occurred via import
+)
+
+BLOCK_TYPES={
+"function": "FunctionBlock",
+"class": "ClassBlock",
+"module": "ModuleBlock",
+}
+
+defdump_symtable(st):
+"""Dump the symtable"""
+out="&SymTable{\n"
+out+='Type:%s,\n'%BLOCK_TYPES[st.get_type()] # Return the type of the symbol table. Possible values are 'class', 'module', and 'function'.
+out+='Name:"%s",\n'%st.get_name() # Return the table’s name. This is the name of the class if the table is for a class, the name of the function if the table is for a function, or 'top' if the table is global (get_type() returns 'module').
+
+out+='Lineno:%s,\n'%st.get_lineno() # Return the number of the first line in the block this table represents.
+out+='Optimized:%s,\n'%dump_bool(st.is_optimized()) # Return True if the locals in this table can be optimized.
+out+='Nested:%s,\n'%dump_bool(st.is_nested()) # Return True if the block is a nested class or function.
+out+='Exec:%s,\n'%dump_bool(st.has_exec()) # Return True if the block uses exec.
+out+='ImportStar:%s,\n'%dump_bool(st.has_import_star()) # Return True if the block uses a starred from-import.
+out+='Symbols: Symbols{\n'
+fornameinst.get_identifiers():
+value=st.lookup(name)
+out+='"%s":%s,\n'% (name, dump_symbol(value))
+out+='},\n'
+# out += 'children:"%s",\n' % st.get_children() # Return a list of the nested symbol tables.
+out+="}"
+returnout
+
+defdump_symbol(s):
+"""Dump a symbol"""
+#class symtable.Symbol
+# An entry in a SymbolTable corresponding to an identifier in the source. The constructor is not public.
+out="Symbol{\n"
+out+='Name:"%s",\n'%s.get_name() # Return the symbol’s name.
+flags= []
+flag_bits=s._Symbol__flags
+forname, maskinDEF_FLAGS:
+if (flag_bits&mask) !=0:
+flags.append(name)
+ifnotflags:
+flags= ["0"]
+out+='Flags:%s,\n'%"|".join(flags)
+
+scope=SCOPES.get(s._Symbol__scope, "scopeUnknown")
+out+='Scope:%s,\n'%scope
+
+# Return a namespace bound to this name.
+ns=s.get_namespaces()
+iflen(ns) ==0:
+pass
+eliflen(ns) ==1:
+out+='Namespace:%s,\n'%dump_symtable(ns[0])
+else:
+raiseAssertionError("More than one namespace")
+out+="}"
+returnout
+
+defescape(x):
+"""Encode strings with backslashes for python/go"""
+returnx.replace('\\', "\\\\").replace('"', r'\"').replace("\n", r'\n').replace("\t", r'\t')
+
+defmain():
+"""Write symtable_data_test.go"""
+path="symtable_data_test.go"
+out= ["""// Test data generated by make_symtable_test.py - do not edit
+
+package compile
+
+import (
+"github.com/ncw/gpython/py"
+)
+
+var symtableTestData = []struct{
+in string
+mode string // exec, eval or single
+out *SymTable
+exceptionType *py.Type
+errString string
+}{"""]
+forxininp:
+source, mode=x[:2]
+iflen(x) >2:
+exc=x[2]
+try:
+table=symtable(source, "<string>", mode)
+exceptexcase:
+error=e.msg
+else:
+raiseValueError("Expecting exception %s"%exc)
+table="nil"
+gostring="nil"
+exc_name="py.%s"%exc.__name__
+else:
+table=symtable(source, "<string>", mode)
+exc_name="nil"
+error=""
+out.append('{"%s", "%s", %s, %s, "%s"},'% (escape(source), mode, dump_symtable(table), exc_name, escape(error)))
+out.append("}\n")
+print("Writing %s"%path)
+withopen(path, "w") asf:
+f.write("\n".join(out))
+f.write("\n")
+subprocess.check_call(["gofmt", "-w", path])
+
+if__name__=="__main__":
+main()