Fix grammar for multiple entry points for compile mode

Author: ncw

ast/ast.go

@@ -1,5 +1,8 @@
 package ast
 
+// FIXME make base AstNode with position in
+// also keep a list of children in the parent node to simplify walking the tree?
+
 import (
 "fmt"
 

parser/grammar.y

@@ -12,21 +12,21 @@ import (
 %}
 
 %union{
- strstring
- objpy.Object
- astast.Ast
- modast.Mod
- stmtast.Stmt
- stmts[]ast.Stmt
- stmts1[]ast.Stmt// nl_or_stmt accumulator
- stmts2[]ast.Stmt// small_stmts accumulator
- stmts3[]ast.Stmt// stmts accumulator
- posast.Pos// kept up to date by the lexer
+ strstring
+ objpy.Object
+ astast.Ast
+ modast.Mod
+ stmtast.Stmt
+ stmts[]ast.Stmt
+ stmts1[]ast.Stmt// nl_or_stmt accumulator
+ stmts2[]ast.Stmt// small_stmts accumulator
+ stmts3[]ast.Stmt// stmts accumulator
+ posast.Pos// kept up to date by the lexer
 }
 
 %type<str>strings
 %type<ast>atom
-%type<mod>inputsfile_input
+%type<mod>inputsfile_inputsingle_inputeval_input
 %type<stmts>simple_stmtstmt
 %type<stmts1>nl_or_stmt
 %type<stmts2>small_stmts
@@ -101,6 +101,8 @@ import (
 
 %token'('')''['']'':'','';''+''-''*''/''|''&''<''>''=''.''%''{''}''^''~''@'
 
+%tokenSINGLE_INPUTFILE_INPUTEVAL_INPUT
+
 // Note: Changing the grammar specified in this file will most likely
 // require corresponding changes in the parser module
 // (../Modules/parsermodule.c). If you can't make the changes to
@@ -119,18 +121,45 @@ import (
 
 %%
 
-// FIXME figure out how to tell the parser to start from a given node
-// inputs: single_input | file_input | eval_input
-// In the mean time just do file_input
-// inputs: single_input | file_input | eval_input
+// Start of grammar. This has 3 pseudo tokens which say which
+// direction through the rest of the grammar we take.
 inputs:
-file_input
+SINGLE_INPUTsingle_input
+{
+ yylex.(*yyLex).mod = $2
+return0
+ }
+|FILE_INPUTfile_input
+{
+ yylex.(*yyLex).mod = $2
+return0
+ }
+|EVAL_INPUTeval_input
 {
- yylex.(*yyLex).mod = $1
+ yylex.(*yyLex).mod = $2
 return0
  }
 
-single_input: NEWLINE|simple_stmt|compound_stmtNEWLINE
+single_input:
+NEWLINE
+{
+$$ = &ast.Interactive{ModBase: ast.ModBase{$<pos>$}}
+ }
+|simple_stmt
+{
+$$ = &ast.Interactive{ModBase: ast.ModBase{$<pos>$}, Body: $1}
+ }
+|compound_stmtNEWLINE
+{
+$$ = &ast.Interactive{ModBase: ast.ModBase{$<pos>$}, Body: []ast.Stmt{$1}}
+ }
+
+//file_input: (NEWLINE|stmt)* ENDMARKER
+file_input:
+nl_or_stmtENDMARKER
+{
+$$ = &ast.Module{ModBase: ast.ModBase{$<pos>$}, Body: $1}
+ }
 
 // (NEWLINE|stmt)*
 nl_or_stmt:
@@ -145,19 +174,15 @@ nl_or_stmt:
 $$ = append($$, $2...)
  }
 
-//file_input: (NEWLINE|stmt)* ENDMARKER
-file_input:
-nl_or_stmtENDMARKER
+//eval_input: testlistNEWLINE* ENDMARKER
+eval_input:
+testlistnlsENDMARKER
 {
-$$ = &ast.Module{ModBase: ast.ModBase{$<pos>$}, Body: $1}
  }
 
 // NEWLINE*
 nls: |nlsNEWLINE
 
-//eval_input: testlistNEWLINE* ENDMARKER
-eval_input: testlistnlsENDMARKER
-
 optional_arglist: |arglist
 
 optional_arglist_call: |'('optional_arglist')'

parser/lexer.go

@@ -46,15 +46,33 @@ type yyLex struct{
 parenthesisint// number of open ( )
 braceint// number of open{}
 mod ast.Mod// output
+startTokenint// initial token to output
 }
 
-funcNewLex(r io.Reader) *yyLex{
+// Create a new lexer
+//
+// The mode argument specifies what kind of code must be compiled; it
+// can be 'exec' if source consists of a sequence of statements,
+// 'eval' if it consists of a single expression, or 'single' if it
+// consists of a single interactive statement
+funcNewLex(r io.Reader, modestring) (*yyLex, error){
 x:=&yyLex{
 reader: bufio.NewReader(r),
 indentStack: []int{0},
 state: readString,
  }
-returnx
+switchmode{
+case"exec":
+x.startToken=FILE_INPUT
+case"eval":
+x.startToken=EVAL_INPUT
+case"single":
+x.startToken=SINGLE_INPUT
+x.interactive=true
+default:
+returnnil, py.ExceptionNewf(py.ValueError, "compile mode must be 'exec', 'eval' or 'single'")
+ }
+returnx, nil
 }
 
 // Refill line
@@ -222,6 +240,9 @@ func init(){
 tokenToString[DEDENT] ="DEDENT"
 tokenToString[STRING] ="STRING"
 tokenToString[NUMBER] ="NUMBER"
+tokenToString[FILE_INPUT] ="FILE_INPUT"
+tokenToString[SINGLE_INPUT] ="SINGLE_INPUT"
+tokenToString[EVAL_INPUT] ="EVAL_INPUT"
 }
 
 // True if there are any open brackets
@@ -309,6 +330,13 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int){
 deferfunc(){fmt.Printf("LEX> %v\n", newLexToken(ret, yylval)) }()
  }
 
+// Return initial token
+ifx.startToken!=eof{
+token:=x.startToken
+x.startToken=eof
+returntoken
+ }
+
 // FIXME keep x.pos up to date
 x.pos.Lineno=42
 x.pos.ColOffset=43
@@ -318,13 +346,14 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int){
 // Read x.line
 x.refill()
 x.state++
-// an empty line while reading interactive input should return a NEWLINE
-ifx.interactive&& (x.line==""||x.line=="\n"){
+ifx.line==""&&x.eof{
+x.state=checkEof
+// an empty line while reading interactive input should return a NEWLINE
 // Don't output NEWLINE if brackets are open
-ifx.openBrackets(){
-continue
+ifx.interactive&&!x.openBrackets(){
+returnNEWLINE
  }
-returnNEWLINE
+continue
  }
 casereadIndent:
 // Read the initial indent and get rid of it
@@ -799,20 +828,26 @@ func SetDebug(level int){
 }
 
 // Parse a file
-funcParse(in io.Reader) (ast.Mod, error){
-lex:=NewLex(in)
+funcParse(in io.Reader, modestring) (ast.Mod, error){
+lex, err:=NewLex(in, mode)
+iferr!=nil{
+returnnil, err
+ }
 yyParse(lex)
 returnlex.mod, lex.ErrorReturn()
 }
 
 // Parse a string
-funcParseString(instring) (ast.Ast, error){
-returnParse(bytes.NewBufferString(in))
+funcParseString(instring, modestring) (ast.Ast, error){
+returnParse(bytes.NewBufferString(in), mode)
 }
 
 // Lex a file only, returning a sequence of tokens
-funcLex(in io.Reader) (ltsLexTokens, errerror){
-lex:=NewLex(in)
+funcLex(in io.Reader, modestring) (ltsLexTokens, errerror){
+lex, err:=NewLex(in, mode)
+iferr!=nil{
+returnnil, err
+ }
 yylval:=yySymType{}
 for{
 ret:=lex.Lex(&yylval)
@@ -827,6 +862,6 @@ func Lex(in io.Reader) (lts LexTokens, err error){
 }
 
 // Lex a string
-funcLexString(instring) (ltsLexTokens, errerror){
-returnLex(bytes.NewBufferString(in))
+funcLexString(instring, modestring) (ltsLexTokens, errerror){
+returnLex(bytes.NewBufferString(in), mode)
 }