@@ -73,153 +73,191 @@ func Extract(fset *token.FileSet, file *ast.File) ([]*Note, error) {
73
73
return notes , nil
74
74
}
75
75
76
- func parse (fset * token.FileSet , base token.Pos , text string ) ([]* Note , error ) {
77
- var scanErr error
78
- s := new (scanner.Scanner ).Init (strings .NewReader (text ))
79
- s .Mode = scanner .GoTokens
80
- s .Whitespace ^= 1 << '\n' // don't skip new lines
81
- s .Error = func (s * scanner.Scanner , msg string ) {
82
- scanErr = fmt .Errorf ("%v:%s" , fset .Position (base + token .Pos (s .Position .Offset )), msg )
76
+ const invalidToken rune = 0
77
+
78
+ type tokens struct {
79
+ scanner scanner.Scanner
80
+ current rune
81
+ err error
82
+ base token.Pos
83
+ }
84
+
85
+ func (t * tokens ) Init (base token.Pos , text string ) * tokens {
86
+ t .base = base
87
+ t .scanner .Init (strings .NewReader (text ))
88
+ t .scanner .Mode = scanner .GoTokens
89
+ t .scanner .Whitespace ^= 1 << '\n' // don't skip new lines
90
+ t .scanner .Error = func (s * scanner.Scanner , msg string ) {
91
+ t .Errorf ("%v" , msg )
83
92
}
84
- notes , err := parseComment (s )
85
- if err != nil {
86
- return nil , fmt .Errorf ("%v:%s" , fset .Position (base + token .Pos (s .Position .Offset )), err )
93
+ return t
94
+ }
95
+
96
+ func (t * tokens ) Consume () string {
97
+ t .current = invalidToken
98
+ return t .scanner .TokenText ()
99
+ }
100
+
101
+ func (t * tokens ) Token () rune {
102
+ if t .err != nil {
103
+ return scanner .EOF
104
+ }
105
+ if t .current == invalidToken {
106
+ t .current = t .scanner .Scan ()
107
+ }
108
+ return t .current
109
+ }
110
+
111
+ func (t * tokens ) Skip (r rune ) int {
112
+ i := 0
113
+ for t .Token () == '\n' {
114
+ t .Consume ()
115
+ i ++
87
116
}
88
- if scanErr != nil {
89
- return nil , scanErr
117
+ return i
118
+ }
119
+
120
+ func (t * tokens ) TokenString () string {
121
+ return scanner .TokenString (t .Token ())
122
+ }
123
+
124
+ func (t * tokens ) Pos () token.Pos {
125
+ return t .base + token .Pos (t .scanner .Position .Offset )
126
+ }
127
+
128
+ func (t * tokens ) Errorf (msg string , args ... interface {}) {
129
+ if t .err != nil {
130
+ return
90
131
}
91
- for _ , n := range notes {
92
- n .Pos += base
132
+ t .err = fmt .Errorf (msg , args ... )
133
+ }
134
+
135
+ func parse (fset * token.FileSet , base token.Pos , text string ) ([]* Note , error ) {
136
+ t := new (tokens ).Init (base , text )
137
+ notes := parseComment (t )
138
+ if t .err != nil {
139
+ return nil , fmt .Errorf ("%v:%s" , fset .Position (t .Pos ()), t .err )
93
140
}
94
141
return notes , nil
95
142
}
96
143
97
- func parseComment (s * scanner. Scanner ) ( []* Note , error ) {
144
+ func parseComment (t * tokens ) []* Note {
98
145
var notes []* Note
99
146
for {
100
- n , err := parseNote (s )
101
- if err != nil {
102
- return nil , err
103
- }
104
- var tok rune = scanner .EOF
105
- if n != nil {
106
- notes = append (notes , n )
107
- tok = s .Scan ()
147
+ t .Skip ('\n' )
148
+ switch t .Token () {
149
+ case scanner .EOF :
150
+ return notes
151
+ case scanner .Ident :
152
+ notes = append (notes , parseNote (t ))
153
+ default :
154
+ t .Errorf ("unexpected %s parsing comment, expect identifier" , t .TokenString ())
155
+ return nil
108
156
}
109
- switch tok {
110
- case ',' , '\n' :
111
- // continue
157
+ switch t .Token () {
112
158
case scanner .EOF :
113
- return notes , nil
159
+ return notes
160
+ case ',' , '\n' :
161
+ t .Consume ()
114
162
default :
115
- return nil , fmt .Errorf ("unexpected %s parsing comment" , scanner .TokenString (tok ))
163
+ t .Errorf ("unexpected %s parsing comment, expect separator" , t .TokenString ())
164
+ return nil
116
165
}
117
166
}
118
167
}
119
168
120
- func parseNote (s * scanner.Scanner ) (* Note , error ) {
121
- tok := s .Scan ()
122
- if tok == scanner .EOF || tok == '\n' {
123
- return nil , nil
124
- }
125
- if tok != scanner .Ident {
126
- return nil , fmt .Errorf ("expected identifier, got %s" , scanner .TokenString (tok ))
127
- }
169
+ func parseNote (t * tokens ) * Note {
128
170
n := & Note {
129
- Pos : token .Pos (s . Position . Offset ),
130
- Name : s . TokenText (),
171
+ Pos : t .Pos (),
172
+ Name : t . Consume (),
131
173
}
132
- switch s .Peek () {
174
+
175
+ switch t .Token () {
133
176
case ',' , '\n' , scanner .EOF :
134
177
// no argument list present
135
- return n , nil
178
+ return n
136
179
case '(' :
137
- s .Scan () // consume the '('
138
- for s .Peek () == '\n' {
139
- s .Scan () // consume all '\n'
140
- }
141
- // special case the empty argument list
142
- if s .Peek () == ')' {
143
- s .Scan () // consume the ')'
144
- n .Args = []interface {}{} // @name() is represented by a non-nil empty slice.
145
- return n , nil
146
- }
147
- // handle a normal argument list
148
- for {
149
- arg , err := parseArgument (s )
150
- if err != nil {
151
- return nil , err
152
- }
153
- n .Args = append (n .Args , arg )
154
- switch s .Peek () {
155
- case ')' :
156
- s .Scan () // consume the ')'
157
- return n , nil
158
- case ',' :
159
- s .Scan () // consume the ','
160
- for s .Peek () == '\n' {
161
- s .Scan () // consume all '\n'
162
- }
163
- default :
164
- return nil , fmt .Errorf ("unexpected %s parsing argument list" , scanner .TokenString (s .Scan ()))
165
- }
166
- }
180
+ n .Args = parseArgumentList (t )
181
+ return n
167
182
default :
168
- return nil , fmt .Errorf ("unexpected %s parsing note" , scanner .TokenString (s .Scan ()))
183
+ t .Errorf ("unexpected %s parsing note" , t .TokenString ())
184
+ return nil
169
185
}
170
186
}
171
187
172
- func parseArgument (s * scanner.Scanner ) (interface {}, error ) {
173
- tok := s .Scan ()
174
- switch tok {
188
+ func parseArgumentList (t * tokens ) []interface {} {
189
+ args := []interface {}{} // @name() is represented by a non-nil empty slice.
190
+ t .Consume () // '('
191
+ t .Skip ('\n' )
192
+ for t .Token () != ')' {
193
+ args = append (args , parseArgument (t ))
194
+ if t .Token () != ',' {
195
+ break
196
+ }
197
+ t .Consume ()
198
+ t .Skip ('\n' )
199
+ }
200
+ if t .Token () != ')' {
201
+ t .Errorf ("unexpected %s parsing argument list" , t .TokenString ())
202
+ return nil
203
+ }
204
+ t .Consume () // ')'
205
+ return args
206
+ }
207
+
208
+ func parseArgument (t * tokens ) interface {} {
209
+ switch t .Token () {
175
210
case scanner .Ident :
176
- v := s . TokenText ()
211
+ v := t . Consume ()
177
212
switch v {
178
213
case "true" :
179
- return true , nil
214
+ return true
180
215
case "false" :
181
- return false , nil
216
+ return false
182
217
case "nil" :
183
- return nil , nil
218
+ return nil
184
219
case "re" :
185
- tok := s .Scan ()
186
- switch tok {
187
- case scanner .String , scanner .RawString :
188
- pattern , _ := strconv .Unquote (s .TokenText ()) // can't fail
189
- re , err := regexp .Compile (pattern )
190
- if err != nil {
191
- return nil , fmt .Errorf ("invalid regular expression %s: %v" , pattern , err )
192
- }
193
- return re , nil
194
- default :
195
- return nil , fmt .Errorf ("re must be followed by string, got %s" , scanner .TokenString (tok ))
220
+ if t .Token () != scanner .String && t .Token () != scanner .RawString {
221
+ t .Errorf ("re must be followed by string, got %s" , t .TokenString ())
222
+ return nil
223
+ }
224
+ pattern , _ := strconv .Unquote (t .Consume ()) // can't fail
225
+ re , err := regexp .Compile (pattern )
226
+ if err != nil {
227
+ t .Errorf ("invalid regular expression %s: %v" , pattern , err )
228
+ return nil
196
229
}
230
+ return re
197
231
default :
198
- return Identifier (v ), nil
232
+ return Identifier (v )
199
233
}
200
234
201
235
case scanner .String , scanner .RawString :
202
- v , _ := strconv .Unquote (s . TokenText ()) // can't fail
203
- return v , nil
236
+ v , _ := strconv .Unquote (t . Consume ()) // can't fail
237
+ return v
204
238
205
239
case scanner .Int :
206
- v , err := strconv .ParseInt (s .TokenText (), 0 , 0 )
240
+ s := t .Consume ()
241
+ v , err := strconv .ParseInt (s , 0 , 0 )
207
242
if err != nil {
208
- return nil , fmt .Errorf ("cannot convert %v to int: %v" , s . TokenText () , err )
243
+ t .Errorf ("cannot convert %v to int: %v" , s , err )
209
244
}
210
- return v , nil
245
+ return v
211
246
212
247
case scanner .Float :
213
- v , err := strconv .ParseFloat (s .TokenText (), 64 )
248
+ s := t .Consume ()
249
+ v , err := strconv .ParseFloat (s , 64 )
214
250
if err != nil {
215
- return nil , fmt .Errorf ("cannot convert %v to float: %v" , s . TokenText () , err )
251
+ t .Errorf ("cannot convert %v to float: %v" , s , err )
216
252
}
217
- return v , nil
253
+ return v
218
254
219
255
case scanner .Char :
220
- return nil , fmt .Errorf ("unexpected char literal %s" , s .TokenText ())
256
+ t .Errorf ("unexpected char literal %s" , t .Consume ())
257
+ return nil
221
258
222
259
default :
223
- return nil , fmt .Errorf ("unexpected %s parsing argument" , scanner .TokenString (tok ))
260
+ t .Errorf ("unexpected %s parsing argument" , t .TokenString ())
261
+ return nil
224
262
}
225
263
}
0 commit comments