@@ -66,6 +66,7 @@ impl Error for ParserError {}
66
66
/// SQL Parser
67
67
pub struct Parser {
68
68
tokens : Vec < Token > ,
69
+ /// The index of the first unprocessed token in `self.tokens`
69
70
index : usize ,
70
71
}
71
72
@@ -558,7 +559,8 @@ impl Parser {
558
559
}
559
560
}
560
561
561
- /// Return first non-whitespace token that has not yet been processed
562
+ /// Return the first non-whitespace token that has not yet been processed
563
+ /// (or None if reached end-of-file)
562
564
pub fn peek_token ( & self ) -> Option < Token > {
563
565
self . peek_nth_token ( 0 )
564
566
}
@@ -567,78 +569,49 @@ impl Parser {
567
569
pub fn peek_nth_token ( & self , mut n : usize ) -> Option < Token > {
568
570
let mut index = self . index ;
569
571
loop {
570
- match self . token_at ( index) {
571
- Some ( Token :: Whitespace ( _) ) => {
572
- index += 1 ;
573
- }
574
- Some ( token) => {
572
+ index += 1 ;
573
+ match self . tokens . get ( index - 1 ) {
574
+ Some ( Token :: Whitespace ( _) ) => continue ,
575
+ non_whitespace => {
575
576
if n == 0 {
576
- return Some ( token ) ;
577
+ return non_whitespace . cloned ( ) ;
577
578
}
578
- index += 1 ;
579
579
n -= 1 ;
580
580
}
581
- None => {
582
- return None ;
583
- }
584
581
}
585
582
}
586
583
}
587
584
588
- /// Get the next token skipping whitespace and increment the token index
585
+ /// Return the first non-whitespace token that has not yet been processed
586
+ /// (or None if reached end-of-file) and mark it as processed. OK to call
587
+ /// repeatedly after reaching EOF.
589
588
pub fn next_token ( & mut self ) -> Option < Token > {
590
589
loop {
591
- match self . next_token_no_skip ( ) {
592
- Some ( Token :: Whitespace ( _) ) => {
593
- continue ;
594
- }
595
- token => {
596
- return token;
597
- }
590
+ self . index += 1 ;
591
+ match self . tokens . get ( self . index - 1 ) {
592
+ Some ( Token :: Whitespace ( _) ) => continue ,
593
+ token => return token. cloned ( ) ,
598
594
}
599
595
}
600
596
}
601
597
602
- /// see the token at this index
603
- fn token_at ( & self , n : usize ) -> Option < Token > {
604
- if let Some ( token) = self . tokens . get ( n) {
605
- Some ( token. clone ( ) )
606
- } else {
607
- None
608
- }
598
+ /// Return the first unprocessed token, possibly whitespace.
599
+ pub fn next_token_no_skip ( & mut self ) -> Option < & Token > {
600
+ self . index += 1 ;
601
+ self . tokens . get ( self . index - 1 )
609
602
}
610
603
611
- pub fn next_token_no_skip ( & mut self ) -> Option < Token > {
612
- if self . index < self . tokens . len ( ) {
613
- self . index += 1 ;
614
- Some ( self . tokens [ self . index - 1 ] . clone ( ) )
615
- } else {
616
- None
617
- }
618
- }
619
-
620
- /// Push back the last one non-whitespace token
621
- pub fn prev_token ( & mut self ) -> Option < Token > {
622
- // TODO: returned value is unused (available via peek_token)
604
+ /// Push back the last one non-whitespace token. Must be called after
605
+ /// `next_token()`, otherwise might panic. OK to call after
606
+ /// `next_token()` indicates an EOF.
607
+ pub fn prev_token ( & mut self ) {
623
608
loop {
624
- match self . prev_token_no_skip ( ) {
625
- Some ( Token :: Whitespace ( _) ) => {
626
- continue ;
627
- }
628
- token => {
629
- return token;
630
- }
631
- }
632
- }
633
- }
634
-
635
- /// Get the previous token and decrement the token index
636
- fn prev_token_no_skip ( & mut self ) -> Option < Token > {
637
- if self . index > 0 {
609
+ assert ! ( self . index > 0 ) ;
638
610
self . index -= 1 ;
639
- Some ( self . tokens [ self . index ] . clone ( ) )
640
- } else {
641
- None
611
+ if let Some ( Token :: Whitespace ( _) ) = self . tokens . get ( self . index ) {
612
+ continue ;
613
+ }
614
+ return ;
642
615
}
643
616
}
644
617
@@ -953,9 +926,7 @@ impl Parser {
953
926
if name. is_some ( ) {
954
927
self . expected ( "PRIMARY, UNIQUE, FOREIGN, or CHECK" , unexpected)
955
928
} else {
956
- if unexpected. is_some ( ) {
957
- self . prev_token ( ) ;
958
- }
929
+ self . prev_token ( ) ;
959
930
Ok ( None )
960
931
}
961
932
}
@@ -1173,8 +1144,7 @@ impl Parser {
1173
1144
reserved_kwds : & [ & str ] ,
1174
1145
) -> Result < Option < SQLIdent > , ParserError > {
1175
1146
let after_as = self . parse_keyword ( "AS" ) ;
1176
- let maybe_alias = self . next_token ( ) ;
1177
- match maybe_alias {
1147
+ match self . next_token ( ) {
1178
1148
// Accept any identifier after `AS` (though many dialects have restrictions on
1179
1149
// keywords that may appear here). If there's no `AS`: don't parse keywords,
1180
1150
// which may start a construct allowed in this position, to be parsed as aliases.
@@ -1192,9 +1162,7 @@ impl Parser {
1192
1162
if after_as {
1193
1163
return self . expected ( "an identifier after AS" , not_an_ident) ;
1194
1164
}
1195
- if not_an_ident. is_some ( ) {
1196
- self . prev_token ( ) ;
1197
- }
1165
+ self . prev_token ( ) ;
1198
1166
Ok ( None ) // no alias found
1199
1167
}
1200
1168
}
@@ -1216,9 +1184,7 @@ impl Parser {
1216
1184
continue ;
1217
1185
}
1218
1186
_ => {
1219
- if token. is_some ( ) {
1220
- self . prev_token ( ) ;
1221
- }
1187
+ self . prev_token ( ) ;
1222
1188
break ;
1223
1189
}
1224
1190
}
@@ -1774,15 +1740,22 @@ mod tests {
1774
1740
1775
1741
#[ test]
1776
1742
fn test_prev_index ( ) {
1777
- let sql = "SELECT version() " ;
1743
+ let sql = "SELECT version" ;
1778
1744
all_dialects ( ) . run_parser_method ( sql, |parser| {
1779
- assert_eq ! ( parser. prev_token( ) , None ) ;
1745
+ assert_eq ! ( parser. peek_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1746
+ assert_eq ! ( parser. next_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1747
+ parser. prev_token ( ) ;
1780
1748
assert_eq ! ( parser. next_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1781
1749
assert_eq ! ( parser. next_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1782
- assert_eq ! ( parser. prev_token( ) , Some ( Token :: make_word ( "version" , None ) ) ) ;
1750
+ parser. prev_token ( ) ;
1783
1751
assert_eq ! ( parser. peek_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1784
- assert_eq ! ( parser. prev_token( ) , Some ( Token :: make_keyword( "SELECT" ) ) ) ;
1785
- assert_eq ! ( parser. prev_token( ) , None ) ;
1752
+ assert_eq ! ( parser. next_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1753
+ assert_eq ! ( parser. peek_token( ) , None ) ;
1754
+ parser. prev_token ( ) ;
1755
+ assert_eq ! ( parser. next_token( ) , Some ( Token :: make_word( "version" , None ) ) ) ;
1756
+ assert_eq ! ( parser. next_token( ) , None ) ;
1757
+ assert_eq ! ( parser. next_token( ) , None ) ;
1758
+ parser. prev_token ( ) ;
1786
1759
} ) ;
1787
1760
}
1788
1761
}
0 commit comments