Skip to content

Commit 895048a

Browse files
ianthehatianthehat
committed
go/expect: rewrite the expectation parser
The expectation langauge is LL(1) but the scanner does not support true lookahead This change adds a true LL(1) token stream and rewrites the parser in terms of it. Also clean up the error handling and use the behaviour to fix all the broken edge cases, and then change the tests to cover the now correct behaviour. Change-Id: If3d602cda490ed2f4732efce400eb8eabce8a8ec Reviewed-on: https://go-review.googlesource.com/c/151998 Run-TryBot: Ian Cottrell <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Rebecca Stambler <[email protected]>
1 parent 1c3d964 commit 895048a

File tree

2 files changed

+148
-103
lines changed

2 files changed

+148
-103
lines changed

‎go/expect/extract.go

Lines changed: 138 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -73,153 +73,191 @@ func Extract(fset *token.FileSet, file *ast.File) ([]*Note, error) {
7373
return notes, nil
7474
}
7575

76-
func parse(fset *token.FileSet, base token.Pos, text string) ([]*Note, error) {
77-
var scanErr error
78-
s := new(scanner.Scanner).Init(strings.NewReader(text))
79-
s.Mode = scanner.GoTokens
80-
s.Whitespace ^= 1 << '\n' // don't skip new lines
81-
s.Error = func(s *scanner.Scanner, msg string) {
82-
scanErr = fmt.Errorf("%v:%s", fset.Position(base+token.Pos(s.Position.Offset)), msg)
76+
const invalidToken rune = 0
77+
78+
type tokens struct {
79+
scanner scanner.Scanner
80+
current rune
81+
err error
82+
base token.Pos
83+
}
84+
85+
func (t *tokens) Init(base token.Pos, text string) *tokens {
86+
t.base = base
87+
t.scanner.Init(strings.NewReader(text))
88+
t.scanner.Mode = scanner.GoTokens
89+
t.scanner.Whitespace ^= 1 << '\n' // don't skip new lines
90+
t.scanner.Error = func(s *scanner.Scanner, msg string) {
91+
t.Errorf("%v", msg)
8392
}
84-
notes, err := parseComment(s)
85-
if err != nil {
86-
return nil, fmt.Errorf("%v:%s", fset.Position(base+token.Pos(s.Position.Offset)), err)
93+
return t
94+
}
95+
96+
func (t *tokens) Consume() string {
97+
t.current = invalidToken
98+
return t.scanner.TokenText()
99+
}
100+
101+
func (t *tokens) Token() rune {
102+
if t.err != nil {
103+
return scanner.EOF
104+
}
105+
if t.current == invalidToken {
106+
t.current = t.scanner.Scan()
107+
}
108+
return t.current
109+
}
110+
111+
func (t *tokens) Skip(r rune) int {
112+
i := 0
113+
for t.Token() == '\n' {
114+
t.Consume()
115+
i++
87116
}
88-
if scanErr != nil {
89-
return nil, scanErr
117+
return i
118+
}
119+
120+
func (t *tokens) TokenString() string {
121+
return scanner.TokenString(t.Token())
122+
}
123+
124+
func (t *tokens) Pos() token.Pos {
125+
return t.base + token.Pos(t.scanner.Position.Offset)
126+
}
127+
128+
func (t *tokens) Errorf(msg string, args ...interface{}) {
129+
if t.err != nil {
130+
return
90131
}
91-
for _, n := range notes {
92-
n.Pos += base
132+
t.err = fmt.Errorf(msg, args...)
133+
}
134+
135+
func parse(fset *token.FileSet, base token.Pos, text string) ([]*Note, error) {
136+
t := new(tokens).Init(base, text)
137+
notes := parseComment(t)
138+
if t.err != nil {
139+
return nil, fmt.Errorf("%v:%s", fset.Position(t.Pos()), t.err)
93140
}
94141
return notes, nil
95142
}
96143

97-
func parseComment(s *scanner.Scanner) ([]*Note, error) {
144+
func parseComment(t *tokens) []*Note {
98145
var notes []*Note
99146
for {
100-
n, err := parseNote(s)
101-
if err != nil {
102-
return nil, err
103-
}
104-
var tok rune = scanner.EOF
105-
if n != nil {
106-
notes = append(notes, n)
107-
tok = s.Scan()
147+
t.Skip('\n')
148+
switch t.Token() {
149+
case scanner.EOF:
150+
return notes
151+
case scanner.Ident:
152+
notes = append(notes, parseNote(t))
153+
default:
154+
t.Errorf("unexpected %s parsing comment, expect identifier", t.TokenString())
155+
return nil
108156
}
109-
switch tok {
110-
case ',', '\n':
111-
// continue
157+
switch t.Token() {
112158
case scanner.EOF:
113-
return notes, nil
159+
return notes
160+
case ',', '\n':
161+
t.Consume()
114162
default:
115-
return nil, fmt.Errorf("unexpected %s parsing comment", scanner.TokenString(tok))
163+
t.Errorf("unexpected %s parsing comment, expect separator", t.TokenString())
164+
return nil
116165
}
117166
}
118167
}
119168

120-
func parseNote(s *scanner.Scanner) (*Note, error) {
121-
tok := s.Scan()
122-
if tok == scanner.EOF || tok == '\n' {
123-
return nil, nil
124-
}
125-
if tok != scanner.Ident {
126-
return nil, fmt.Errorf("expected identifier, got %s", scanner.TokenString(tok))
127-
}
169+
func parseNote(t *tokens) *Note {
128170
n := &Note{
129-
Pos: token.Pos(s.Position.Offset),
130-
Name: s.TokenText(),
171+
Pos: t.Pos(),
172+
Name: t.Consume(),
131173
}
132-
switch s.Peek() {
174+
175+
switch t.Token() {
133176
case ',', '\n', scanner.EOF:
134177
// no argument list present
135-
return n, nil
178+
return n
136179
case '(':
137-
s.Scan() // consume the '('
138-
for s.Peek() == '\n' {
139-
s.Scan() // consume all '\n'
140-
}
141-
// special case the empty argument list
142-
if s.Peek() == ')' {
143-
s.Scan() // consume the ')'
144-
n.Args = []interface{}{} // @name() is represented by a non-nil empty slice.
145-
return n, nil
146-
}
147-
// handle a normal argument list
148-
for {
149-
arg, err := parseArgument(s)
150-
if err != nil {
151-
return nil, err
152-
}
153-
n.Args = append(n.Args, arg)
154-
switch s.Peek() {
155-
case ')':
156-
s.Scan() // consume the ')'
157-
return n, nil
158-
case ',':
159-
s.Scan() // consume the ','
160-
for s.Peek() == '\n' {
161-
s.Scan() // consume all '\n'
162-
}
163-
default:
164-
return nil, fmt.Errorf("unexpected %s parsing argument list", scanner.TokenString(s.Scan()))
165-
}
166-
}
180+
n.Args = parseArgumentList(t)
181+
return n
167182
default:
168-
return nil, fmt.Errorf("unexpected %s parsing note", scanner.TokenString(s.Scan()))
183+
t.Errorf("unexpected %s parsing note", t.TokenString())
184+
return nil
169185
}
170186
}
171187

172-
func parseArgument(s *scanner.Scanner) (interface{}, error) {
173-
tok := s.Scan()
174-
switch tok {
188+
func parseArgumentList(t *tokens) []interface{} {
189+
args := []interface{}{} // @name() is represented by a non-nil empty slice.
190+
t.Consume() // '('
191+
t.Skip('\n')
192+
for t.Token() != ')' {
193+
args = append(args, parseArgument(t))
194+
if t.Token() != ',' {
195+
break
196+
}
197+
t.Consume()
198+
t.Skip('\n')
199+
}
200+
if t.Token() != ')' {
201+
t.Errorf("unexpected %s parsing argument list", t.TokenString())
202+
return nil
203+
}
204+
t.Consume() // ')'
205+
return args
206+
}
207+
208+
func parseArgument(t *tokens) interface{} {
209+
switch t.Token() {
175210
case scanner.Ident:
176-
v := s.TokenText()
211+
v := t.Consume()
177212
switch v {
178213
case "true":
179-
return true, nil
214+
return true
180215
case "false":
181-
return false, nil
216+
return false
182217
case "nil":
183-
return nil, nil
218+
return nil
184219
case "re":
185-
tok := s.Scan()
186-
switch tok {
187-
case scanner.String, scanner.RawString:
188-
pattern, _ := strconv.Unquote(s.TokenText()) // can't fail
189-
re, err := regexp.Compile(pattern)
190-
if err != nil {
191-
return nil, fmt.Errorf("invalid regular expression %s: %v", pattern, err)
192-
}
193-
return re, nil
194-
default:
195-
return nil, fmt.Errorf("re must be followed by string, got %s", scanner.TokenString(tok))
220+
if t.Token() != scanner.String && t.Token() != scanner.RawString {
221+
t.Errorf("re must be followed by string, got %s", t.TokenString())
222+
return nil
223+
}
224+
pattern, _ := strconv.Unquote(t.Consume()) // can't fail
225+
re, err := regexp.Compile(pattern)
226+
if err != nil {
227+
t.Errorf("invalid regular expression %s: %v", pattern, err)
228+
return nil
196229
}
230+
return re
197231
default:
198-
return Identifier(v), nil
232+
return Identifier(v)
199233
}
200234

201235
case scanner.String, scanner.RawString:
202-
v, _ := strconv.Unquote(s.TokenText()) // can't fail
203-
return v, nil
236+
v, _ := strconv.Unquote(t.Consume()) // can't fail
237+
return v
204238

205239
case scanner.Int:
206-
v, err := strconv.ParseInt(s.TokenText(), 0, 0)
240+
s := t.Consume()
241+
v, err := strconv.ParseInt(s, 0, 0)
207242
if err != nil {
208-
return nil, fmt.Errorf("cannot convert %v to int: %v", s.TokenText(), err)
243+
t.Errorf("cannot convert %v to int: %v", s, err)
209244
}
210-
return v, nil
245+
return v
211246

212247
case scanner.Float:
213-
v, err := strconv.ParseFloat(s.TokenText(), 64)
248+
s := t.Consume()
249+
v, err := strconv.ParseFloat(s, 64)
214250
if err != nil {
215-
return nil, fmt.Errorf("cannot convert %v to float: %v", s.TokenText(), err)
251+
t.Errorf("cannot convert %v to float: %v", s, err)
216252
}
217-
return v, nil
253+
return v
218254

219255
case scanner.Char:
220-
return nil, fmt.Errorf("unexpected char literal %s", s.TokenText())
256+
t.Errorf("unexpected char literal %s", t.Consume())
257+
return nil
221258

222259
default:
223-
return nil, fmt.Errorf("unexpected %s parsing argument", scanner.TokenString(tok))
260+
t.Errorf("unexpected %s parsing argument", t.TokenString())
261+
return nil
224262
}
225263
}

‎go/expect/testdata/test.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ func someFunc(a, b int) int {
2525
}
2626

2727
// And some extra checks for interesting action parameters
28-
//@check(αSimpleMarker)
29-
//@check(StringAndInt, "Number %d", 12)
30-
//@check(Bool, true)
28+
// Also checks for multi-line expectations
29+
/*@
30+
check(αSimpleMarker)
31+
check(StringAndInt,
32+
"Number %d",
33+
12,
34+
)
35+
36+
check(Bool, true)
37+
*/

0 commit comments

Comments
 (0)