@@ -895,7 +895,7 @@ impl<'a> Tokenizer<'a> {
895
895
} ;
896
896
897
897
let mut location = state. location ( ) ;
898
- while let Some ( token) = self . next_token ( & mut state, buf . last ( ) . map ( |t| & t . token ) ) ? {
898
+ while let Some ( token) = self . next_token ( & mut state) ? {
899
899
let span = location. span_to ( state. location ( ) ) ;
900
900
901
901
buf. push ( TokenWithSpan { token, span } ) ;
@@ -932,11 +932,7 @@ impl<'a> Tokenizer<'a> {
932
932
}
933
933
934
934
/// Get the next token or return None
935
- fn next_token (
936
- & self ,
937
- chars : & mut State ,
938
- prev_token : Option < & Token > ,
939
- ) -> Result < Option < Token > , TokenizerError > {
935
+ fn next_token ( & self , chars : & mut State ) -> Result < Option < Token > , TokenizerError > {
940
936
match chars. peek ( ) {
941
937
Some ( & ch) => match ch {
942
938
' ' => self . consume_and_return ( chars, Token :: Whitespace ( Whitespace :: Space ) ) ,
@@ -1215,28 +1211,17 @@ impl<'a> Tokenizer<'a> {
1215
1211
chars. next ( ) ;
1216
1212
}
1217
1213
1218
- // If the dialect supports identifiers that start with a numeric prefix
1219
- // and we have now consumed a dot, check if the previous token was a Word.
1220
- // If so, what follows is definitely not part of a decimal number and
1221
- // we should yield the dot as a dedicated token so compound identifiers
1222
- // starting with digits can be parsed correctly.
1223
- if s == "." && self . dialect . supports_numeric_prefix ( ) {
1224
- if let Some ( Token :: Word ( _) ) = prev_token {
1225
- return Ok ( Some ( Token :: Period ) ) ;
1226
- }
1227
- }
1228
-
1229
1214
// Consume fractional digits.
1230
1215
s += & peeking_next_take_while ( chars, |ch, next_ch| {
1231
1216
ch. is_ascii_digit ( ) || is_number_separator ( ch, next_ch)
1232
1217
} ) ;
1233
1218
1234
- // No fraction -> Token::Period
1219
+ // No fraction -> Token::Period.
1235
1220
if s == "." {
1236
1221
return Ok ( Some ( Token :: Period ) ) ;
1237
1222
}
1238
1223
1239
- // Parse exponent as number
1224
+ // Parse exponent, if present.
1240
1225
let mut exponent_part = String :: new ( ) ;
1241
1226
if chars. peek ( ) == Some ( & 'e' ) || chars. peek ( ) == Some ( & 'E' ) {
1242
1227
let mut char_clone = chars. peekable . clone ( ) ;
@@ -1269,20 +1254,14 @@ impl<'a> Tokenizer<'a> {
1269
1254
// If the dialect supports identifiers that start with a numeric prefix,
1270
1255
// we need to check if the value is in fact an identifier and must thus
1271
1256
// be tokenized as a word.
1272
- if self . dialect . supports_numeric_prefix ( ) {
1273
- if exponent_part. is_empty ( ) {
1274
- // If it is not a number with an exponent, it may be
1275
- // an identifier starting with digits.
1276
- let word =
1277
- peeking_take_while ( chars, |ch| self . dialect . is_identifier_part ( ch) ) ;
1278
-
1279
- if !word. is_empty ( ) {
1280
- s += word. as_str ( ) ;
1281
- return Ok ( Some ( Token :: make_word ( s. as_str ( ) , None ) ) ) ;
1282
- }
1283
- } else if prev_token == Some ( & Token :: Period ) {
1284
- // If the previous token was a period, thus not belonging to a number,
1285
- // the value we have is part of an identifier.
1257
+ if self . dialect . supports_numeric_prefix ( ) && exponent_part. is_empty ( ) {
1258
+ // If it is not a number with an exponent, it may be
1259
+ // an identifier starting with digits.
1260
+ let word =
1261
+ peeking_take_while ( chars, |ch| self . dialect . is_identifier_part ( ch) ) ;
1262
+
1263
+ if !word. is_empty ( ) {
1264
+ s += word. as_str ( ) ;
1286
1265
return Ok ( Some ( Token :: make_word ( s. as_str ( ) , None ) ) ) ;
1287
1266
}
1288
1267
}
@@ -3985,31 +3964,4 @@ mod tests {
3985
3964
] ,
3986
3965
) ;
3987
3966
}
3988
-
3989
- #[ test]
3990
- fn test_tokenize_identifiers_numeric_prefix ( ) {
3991
- all_dialects_where ( |dialect| dialect. supports_numeric_prefix ( ) )
3992
- . tokenizes_to ( "123abc" , vec ! [ Token :: make_word( "123abc" , None ) ] ) ;
3993
-
3994
- all_dialects_where ( |dialect| dialect. supports_numeric_prefix ( ) )
3995
- . tokenizes_to ( "12e34" , vec ! [ Token :: Number ( "12e34" . to_string( ) , false ) ] ) ;
3996
-
3997
- all_dialects_where ( |dialect| dialect. supports_numeric_prefix ( ) ) . tokenizes_to (
3998
- "t.12e34" ,
3999
- vec ! [
4000
- Token :: make_word( "t" , None ) ,
4001
- Token :: Period ,
4002
- Token :: make_word( "12e34" , None ) ,
4003
- ] ,
4004
- ) ;
4005
-
4006
- all_dialects_where ( |dialect| dialect. supports_numeric_prefix ( ) ) . tokenizes_to (
4007
- "t.1two3" ,
4008
- vec ! [
4009
- Token :: make_word( "t" , None ) ,
4010
- Token :: Period ,
4011
- Token :: make_word( "1two3" , None ) ,
4012
- ] ,
4013
- ) ;
4014
- }
4015
3967
}
0 commit comments