Parse string literals into AST

Author: ncw

ast/ast.go

@@ -11,7 +11,6 @@ import (
 
 typeIdentifier py.String
 typeString py.String
-typePyBytes py.Bytes
 typeObject py.Object
 typeSingleton py.Object
 
@@ -516,7 +515,7 @@ type Str struct{
 
 typeBytesstruct{
 ExprBase
-SPyBytes
+Spy.Bytes
 }
 
 typeNameConstantstruct{

parser/grammar.y

@@ -17,6 +17,10 @@ import (
 
 // FIXME can put code blocks in not just at the end - help with list initialisation
 
+// FIXME is the expStack needed at all? Aren't the yylval put into a
+// stack anyway by yacc? And in rsc cc he sets yylval to empty every
+// lex.
+
 // A stack of []ast.Expr
 type exprsStack [][]ast.Expr
 
@@ -81,7 +85,7 @@ func (es *stmtsStack) Add(stmt ...ast.Stmt){
  comma bool
 }
 
-%type<str>strings
+%type<obj>strings
 %type<mod>inputsfile_inputsingle_inputeval_input
 %type<stmts>simple_stmtstmt
 %type<stmtsStack>nl_or_stmtsmall_stmtsstmts
@@ -99,8 +103,8 @@ func (es *stmtsStack) Add(stmt ...ast.Stmt){
 %token<str>NAME
 %tokenINDENT
 %tokenDEDENT
-%token<str>STRING
-%token<str>NUMBER
+%token<obj>STRING
+%token<obj>NUMBER
 
 %tokenPLINGEQ // !=
 %tokenPERCEQ // %=
@@ -1051,9 +1055,27 @@ trailers:
 
 strings:
 STRING
+{
+$$ = $1
+ }
 |stringsSTRING
 {
-$$ += $2
+switch a := $$.(type){
+case py.String:
+switch b := $2.(type){
+case py.String:
+$$ = a + b
+default:
+ yylex.Error("SyntaxError: cannot mix string and nonstring literals")
+ }
+case py.Bytes:
+switch b := $2.(type){
+case py.Bytes:
+$$ = append(a, b...)
+default:
+ yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
+ }
+ }
  }
 
 atom:
@@ -1100,8 +1122,14 @@ atom:
  }
 |strings
 {
-// FIXME
-$$ = nil
+switch s := $1.(type){
+case py.String:
+$$ = &ast.Str{ExprBase: ast.ExprBase{$<pos>$}, S: s}
+case py.Bytes:
+$$ = &ast.Bytes{ExprBase: ast.ExprBase{$<pos>$}, S: s}
+default:
+panic("not Bytes or String in strings")
+ }
  }
 | ELIPSIS
 {

parser/grammar_test.go

@@ -25,6 +25,10 @@ func TestGrammar(t *testing.T){
 {"None\n", "eval", "Expression(body=NameConstant(value=None))"},
 {"...", "eval", "Expression(body=Ellipsis())"},
 {"abc123", "eval", "Expression(body=Name(id='abc123', ctx=Load()))"},
+{"\"abc\"", "eval", "Expression(body=Str(s='abc'))"},
+{"\"abc\"\"\"\"123\"\"\"", "eval", "Expression(body=Str(s='abc123'))"},
+{"b'abc'", "eval", "Expression(body=Bytes(s=b'abc'))"},
+{"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))"},
 // END TESTS
  }{
 Ast, err:=ParseString(test.in, test.mode)

parser/make_grammar_test.py

@@ -17,6 +17,10 @@
  ("None\n", "eval"),
  ("...", "eval"),
  ("abc123", "eval"),
+ ('"abc"', "eval"),
+ ('"abc" """123"""', "eval"),
+ ("b'abc'", "eval"),
+ ("b'abc' b'''123'''", "eval"),
 ]
 
 defdump(source, mode):
@@ -26,7 +30,7 @@ def dump(source, mode):
 
 defescape(x):
 """Encode strings with backslashes for python/go"""
-returnx.encode("unicode_escape").decode("utf-8")
+returnx.replace('\\', "\\\\").replace('"', r'\"').replace("\n", r'\n').replace("\t", r'\t')
 
 defmain():
 """Read in grammar_test.go, and re-write the tests section"""