@@ -66,6 +66,7 @@ impl Error for ParserError {}
66
66
/// SQL Parser
67
67
pub struct Parser {
68
68
tokens : Vec < Token > ,
69
+ /// The index of the first unprocessed token in `self.tokens`
69
70
index : usize ,
70
71
}
71
72
@@ -558,7 +559,8 @@ impl Parser {
558
559
}
559
560
}
560
561
561
- /// Return first non-whitespace token that has not yet been processed
562
+ /// Return the first non-whitespace token that has not yet been processed
563
+ /// (or None if reached end-of-file)
562
564
pub fn peek_token ( & self ) -> Option < Token > {
563
565
self . peek_nth_token ( 0 )
564
566
}
@@ -567,53 +569,48 @@ impl Parser {
567
569
pub fn peek_nth_token ( & self , mut n : usize ) -> Option < Token > {
568
570
let mut index = self . index ;
569
571
loop {
570
- match self . tokens . get ( index) {
571
- Some ( Token :: Whitespace ( _) ) => {
572
- index += 1 ;
573
- }
574
- Some ( token) => {
572
+ index += 1 ;
573
+ match self . tokens . get ( index - 1 ) {
574
+ Some ( Token :: Whitespace ( _) ) => continue ,
575
+ non_whitespace => {
575
576
if n == 0 {
576
- return Some ( token . clone ( ) ) ;
577
+ return non_whitespace . cloned ( ) ;
577
578
}
578
- index += 1 ;
579
579
n -= 1 ;
580
580
}
581
- None => {
582
- return None ;
583
- }
584
581
}
585
582
}
586
583
}
587
584
588
- /// Get the next token skipping whitespace and increment the token index
585
+ /// Return the first non-whitespace token that has not yet been processed
586
+ /// (or None if reached end-of-file) and mark it as processed. OK to call
587
+ /// repeatedly after reaching EOF.
589
588
pub fn next_token ( & mut self ) -> Option < Token > {
590
589
loop {
591
- match self . next_token_no_skip ( ) {
590
+ self . index += 1 ;
591
+ match self . tokens . get ( self . index - 1 ) {
592
592
Some ( Token :: Whitespace ( _) ) => continue ,
593
593
token => return token. cloned ( ) ,
594
594
}
595
595
}
596
596
}
597
597
598
+ /// Return the first unprocessed token, possibly whitespace.
598
599
pub fn next_token_no_skip ( & mut self ) -> Option < & Token > {
599
- if self . index < self . tokens . len ( ) {
600
- self . index += 1 ;
601
- Some ( & self . tokens [ self . index - 1 ] )
602
- } else {
603
- None
604
- }
600
+ self . index += 1 ;
601
+ self . tokens . get ( self . index - 1 )
605
602
}
606
603
607
- /// Push back the last one non-whitespace token
604
+ /// Push back the last one non-whitespace token. Must be called after
605
+ /// `next_token()`, otherwise might panic. OK to call after
606
+ /// `next_token()` indicates an EOF.
608
607
pub fn prev_token ( & mut self ) {
609
608
loop {
610
609
assert ! ( self . index > 0 ) ;
611
- if self . index > 0 {
612
- self . index -= 1 ;
613
- if let Token :: Whitespace ( _) = & self . tokens [ self . index ] {
614
- continue ;
615
- }
616
- } ;
610
+ self . index -= 1 ;
611
+ if let Some ( Token :: Whitespace ( _) ) = self . tokens . get ( self . index ) {
612
+ continue ;
613
+ }
617
614
return ;
618
615
}
619
616
}
@@ -929,9 +926,7 @@ impl Parser {
929
926
if name. is_some ( ) {
930
927
self . expected ( "PRIMARY, UNIQUE, FOREIGN, or CHECK" , unexpected)
931
928
} else {
932
- if unexpected. is_some ( ) {
933
- self . prev_token ( ) ;
934
- }
929
+ self . prev_token ( ) ;
935
930
Ok ( None )
936
931
}
937
932
}
@@ -1149,8 +1144,7 @@ impl Parser {
1149
1144
reserved_kwds : & [ & str ] ,
1150
1145
) -> Result < Option < SQLIdent > , ParserError > {
1151
1146
let after_as = self . parse_keyword ( "AS" ) ;
1152
- let maybe_alias = self . next_token ( ) ;
1153
- match maybe_alias {
1147
+ match self . next_token ( ) {
1154
1148
// Accept any identifier after `AS` (though many dialects have restrictions on
1155
1149
// keywords that may appear here). If there's no `AS`: don't parse keywords,
1156
1150
// which may start a construct allowed in this position, to be parsed as aliases.
@@ -1168,9 +1162,7 @@ impl Parser {
1168
1162
if after_as {
1169
1163
return self . expected ( "an identifier after AS" , not_an_ident) ;
1170
1164
}
1171
- if not_an_ident. is_some ( ) {
1172
- self . prev_token ( ) ;
1173
- }
1165
+ self . prev_token ( ) ;
1174
1166
Ok ( None ) // no alias found
1175
1167
}
1176
1168
}
@@ -1192,9 +1184,7 @@ impl Parser {
1192
1184
continue ;
1193
1185
}
1194
1186
_ => {
1195
- if token. is_some ( ) {
1196
- self . prev_token ( ) ;
1197
- }
1187
+ self . prev_token ( ) ;
1198
1188
break ;
1199
1189
}
1200
1190
}
@@ -1750,14 +1740,22 @@ mod tests {
1750
1740
1751
1741
#[ test]
1752
1742
fn test_prev_index ( ) {
1753
- let sql = "SELECT version() " ;
1743
+ let sql = "SELECT version" ;
1754
1744
all_dialects ( ) . run_parser_method ( sql, |parser| {
1745
+ assert_eq ! ( parser. peek_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1746
+ assert_eq ! ( parser. next_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1747
+ parser. prev_token ( ) ;
1755
1748
assert_eq ! ( parser. next_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1756
1749
assert_eq ! ( parser. next_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1757
1750
parser. prev_token ( ) ;
1758
1751
assert_eq ! ( parser. peek_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1752
+ assert_eq ! ( parser. next_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1753
+ assert_eq ! ( parser. peek_token( ) , None ) ;
1754
+ parser. prev_token ( ) ;
1755
+ assert_eq ! ( parser. next_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1756
+ assert_eq ! ( parser. next_token( ) , None ) ;
1757
+ assert_eq ! ( parser. next_token( ) , None ) ;
1759
1758
parser. prev_token ( ) ;
1760
- assert_eq ! ( parser. peek_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1761
1759
} ) ;
1762
1760
}
1763
1761
}
0 commit comments