Skip to content

Commit e026257

Browse files
committed
Allow calling prev_token() after EOF
Before this `next_token()` would only increment the index when returning `Some(token)`. This means that the caller wishing to rewind must be careful not to call `prev_token()` on EOF (`None`), while not forgetting to call it for `Some`. Not doing this resulted in bugs in the undertested code that does error handling. After making this mistake several times, I'm changing `next_token()` / `prev_token()` so that calling `next_token(); prev_token(); next_token()` returns the same token in the first and the last invocation.
1 parent 1227fdd commit e026257

File tree

1 file changed

+37
-39
lines changed

1 file changed

+37
-39
lines changed

src/sqlparser.rs

+37-39
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ impl Error for ParserError {}
6666
/// SQL Parser
6767
pub struct Parser {
6868
tokens: Vec<Token>,
69+
/// The index of the first unprocessed token in `self.tokens`
6970
index: usize,
7071
}
7172

@@ -558,7 +559,8 @@ impl Parser {
558559
}
559560
}
560561

561-
/// Return first non-whitespace token that has not yet been processed
562+
/// Return the first non-whitespace token that has not yet been processed
563+
/// (or None if reached end-of-file)
562564
pub fn peek_token(&self) -> Option<Token> {
563565
self.peek_nth_token(0)
564566
}
@@ -567,53 +569,48 @@ impl Parser {
567569
pub fn peek_nth_token(&self, mut n: usize) -> Option<Token> {
568570
let mut index = self.index;
569571
loop {
570-
match self.tokens.get(index) {
571-
Some(Token::Whitespace(_)) => {
572-
index += 1;
573-
}
574-
Some(token) => {
572+
index += 1;
573+
match self.tokens.get(index - 1) {
574+
Some(Token::Whitespace(_)) => continue,
575+
non_whitespace => {
575576
if n == 0 {
576-
return Some(token.clone());
577+
return non_whitespace.cloned();
577578
}
578-
index += 1;
579579
n -= 1;
580580
}
581-
None => {
582-
return None;
583-
}
584581
}
585582
}
586583
}
587584

588-
/// Get the next token skipping whitespace and increment the token index
585+
/// Return the first non-whitespace token that has not yet been processed
586+
/// (or None if reached end-of-file) and mark it as processed. OK to call
587+
/// repeatedly after reaching EOF.
589588
pub fn next_token(&mut self) -> Option<Token> {
590589
loop {
591-
match self.next_token_no_skip() {
590+
self.index += 1;
591+
match self.tokens.get(self.index - 1) {
592592
Some(Token::Whitespace(_)) => continue,
593593
token => return token.cloned(),
594594
}
595595
}
596596
}
597597

598+
/// Return the first unprocessed token, possibly whitespace.
598599
pub fn next_token_no_skip(&mut self) -> Option<&Token> {
599-
if self.index < self.tokens.len() {
600-
self.index += 1;
601-
Some(&self.tokens[self.index - 1])
602-
} else {
603-
None
604-
}
600+
self.index += 1;
601+
self.tokens.get(self.index - 1)
605602
}
606603

607-
/// Push back the last one non-whitespace token
604+
/// Push back the last one non-whitespace token. Must be called after
605+
/// `next_token()`, otherwise might panic. OK to call after
606+
/// `next_token()` indicates an EOF.
608607
pub fn prev_token(&mut self) {
609608
loop {
610609
assert!(self.index > 0);
611-
if self.index > 0 {
612-
self.index -= 1;
613-
if let Token::Whitespace(_) = &self.tokens[self.index] {
614-
continue;
615-
}
616-
};
610+
self.index -= 1;
611+
if let Some(Token::Whitespace(_)) = self.tokens.get(self.index) {
612+
continue;
613+
}
617614
return;
618615
}
619616
}
@@ -929,9 +926,7 @@ impl Parser {
929926
if name.is_some() {
930927
self.expected("PRIMARY, UNIQUE, FOREIGN, or CHECK", unexpected)
931928
} else {
932-
if unexpected.is_some() {
933-
self.prev_token();
934-
}
929+
self.prev_token();
935930
Ok(None)
936931
}
937932
}
@@ -1149,8 +1144,7 @@ impl Parser {
11491144
reserved_kwds: &[&str],
11501145
) -> Result<Option<SQLIdent>, ParserError> {
11511146
let after_as = self.parse_keyword("AS");
1152-
let maybe_alias = self.next_token();
1153-
match maybe_alias {
1147+
match self.next_token() {
11541148
// Accept any identifier after `AS` (though many dialects have restrictions on
11551149
// keywords that may appear here). If there's no `AS`: don't parse keywords,
11561150
// which may start a construct allowed in this position, to be parsed as aliases.
@@ -1168,9 +1162,7 @@ impl Parser {
11681162
if after_as {
11691163
return self.expected("an identifier after AS", not_an_ident);
11701164
}
1171-
if not_an_ident.is_some() {
1172-
self.prev_token();
1173-
}
1165+
self.prev_token();
11741166
Ok(None) // no alias found
11751167
}
11761168
}
@@ -1192,9 +1184,7 @@ impl Parser {
11921184
continue;
11931185
}
11941186
_ => {
1195-
if token.is_some() {
1196-
self.prev_token();
1197-
}
1187+
self.prev_token();
11981188
break;
11991189
}
12001190
}
@@ -1750,14 +1740,22 @@ mod tests {
17501740

17511741
#[test]
17521742
fn test_prev_index() {
1753-
let sql = "SELECT version()";
1743+
let sql = "SELECT version";
17541744
all_dialects().run_parser_method(sql, |parser| {
1745+
assert_eq!(parser.peek_token(), Some(Token::make_keyword("SELECT")));
1746+
assert_eq!(parser.next_token(), Some(Token::make_keyword("SELECT")));
1747+
parser.prev_token();
17551748
assert_eq!(parser.next_token(), Some(Token::make_keyword("SELECT")));
17561749
assert_eq!(parser.next_token(), Some(Token::make_word("version", None)));
17571750
parser.prev_token();
17581751
assert_eq!(parser.peek_token(), Some(Token::make_word("version", None)));
1752+
assert_eq!(parser.next_token(), Some(Token::make_word("version", None)));
1753+
assert_eq!(parser.peek_token(), None);
1754+
parser.prev_token();
1755+
assert_eq!(parser.next_token(), Some(Token::make_word("version", None)));
1756+
assert_eq!(parser.next_token(), None);
1757+
assert_eq!(parser.next_token(), None);
17591758
parser.prev_token();
1760-
assert_eq!(parser.peek_token(), Some(Token::make_keyword("SELECT")));
17611759
});
17621760
}
17631761
}

0 commit comments

Comments
 (0)