Skip to content

Commit 2308c1c

Browse files
authored
Merge pull request #86 from nickolay/pr/token-refactor
Internal improvements to Parser::next_token/prev_token This reduces the number of helper functions used by next_token()/prev_token() while slightly improving performance and reducing the chances of coding errors when using prev_token() after hitting end-of-file.
2 parents ebb82b8 + e026257 commit 2308c1c

File tree

1 file changed

+44
-71
lines changed

1 file changed

+44
-71
lines changed

src/sqlparser.rs

+44-71
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ impl Error for ParserError {}
6666
/// SQL Parser
6767
pub struct Parser {
6868
tokens: Vec<Token>,
69+
/// The index of the first unprocessed token in `self.tokens`
6970
index: usize,
7071
}
7172

@@ -558,7 +559,8 @@ impl Parser {
558559
}
559560
}
560561

561-
/// Return first non-whitespace token that has not yet been processed
562+
/// Return the first non-whitespace token that has not yet been processed
563+
/// (or None if reached end-of-file)
562564
pub fn peek_token(&self) -> Option<Token> {
563565
self.peek_nth_token(0)
564566
}
@@ -567,78 +569,49 @@ impl Parser {
567569
pub fn peek_nth_token(&self, mut n: usize) -> Option<Token> {
568570
let mut index = self.index;
569571
loop {
570-
match self.token_at(index) {
571-
Some(Token::Whitespace(_)) => {
572-
index += 1;
573-
}
574-
Some(token) => {
572+
index += 1;
573+
match self.tokens.get(index - 1) {
574+
Some(Token::Whitespace(_)) => continue,
575+
non_whitespace => {
575576
if n == 0 {
576-
return Some(token);
577+
return non_whitespace.cloned();
577578
}
578-
index += 1;
579579
n -= 1;
580580
}
581-
None => {
582-
return None;
583-
}
584581
}
585582
}
586583
}
587584

588-
/// Get the next token skipping whitespace and increment the token index
585+
/// Return the first non-whitespace token that has not yet been processed
586+
/// (or None if reached end-of-file) and mark it as processed. OK to call
587+
/// repeatedly after reaching EOF.
589588
pub fn next_token(&mut self) -> Option<Token> {
590589
loop {
591-
match self.next_token_no_skip() {
592-
Some(Token::Whitespace(_)) => {
593-
continue;
594-
}
595-
token => {
596-
return token;
597-
}
590+
self.index += 1;
591+
match self.tokens.get(self.index - 1) {
592+
Some(Token::Whitespace(_)) => continue,
593+
token => return token.cloned(),
598594
}
599595
}
600596
}
601597

602-
/// see the token at this index
603-
fn token_at(&self, n: usize) -> Option<Token> {
604-
if let Some(token) = self.tokens.get(n) {
605-
Some(token.clone())
606-
} else {
607-
None
608-
}
598+
/// Return the first unprocessed token, possibly whitespace.
599+
pub fn next_token_no_skip(&mut self) -> Option<&Token> {
600+
self.index += 1;
601+
self.tokens.get(self.index - 1)
609602
}
610603

611-
pub fn next_token_no_skip(&mut self) -> Option<Token> {
612-
if self.index < self.tokens.len() {
613-
self.index += 1;
614-
Some(self.tokens[self.index - 1].clone())
615-
} else {
616-
None
617-
}
618-
}
619-
620-
/// Push back the last one non-whitespace token
621-
pub fn prev_token(&mut self) -> Option<Token> {
622-
// TODO: returned value is unused (available via peek_token)
604+
/// Push back the last one non-whitespace token. Must be called after
605+
/// `next_token()`, otherwise might panic. OK to call after
606+
/// `next_token()` indicates an EOF.
607+
pub fn prev_token(&mut self) {
623608
loop {
624-
match self.prev_token_no_skip() {
625-
Some(Token::Whitespace(_)) => {
626-
continue;
627-
}
628-
token => {
629-
return token;
630-
}
631-
}
632-
}
633-
}
634-
635-
/// Get the previous token and decrement the token index
636-
fn prev_token_no_skip(&mut self) -> Option<Token> {
637-
if self.index > 0 {
609+
assert!(self.index > 0);
638610
self.index -= 1;
639-
Some(self.tokens[self.index].clone())
640-
} else {
641-
None
611+
if let Some(Token::Whitespace(_)) = self.tokens.get(self.index) {
612+
continue;
613+
}
614+
return;
642615
}
643616
}
644617

@@ -953,9 +926,7 @@ impl Parser {
953926
if name.is_some() {
954927
self.expected("PRIMARY, UNIQUE, FOREIGN, or CHECK", unexpected)
955928
} else {
956-
if unexpected.is_some() {
957-
self.prev_token();
958-
}
929+
self.prev_token();
959930
Ok(None)
960931
}
961932
}
@@ -1173,8 +1144,7 @@ impl Parser {
11731144
reserved_kwds: &[&str],
11741145
) -> Result<Option<SQLIdent>, ParserError> {
11751146
let after_as = self.parse_keyword("AS");
1176-
let maybe_alias = self.next_token();
1177-
match maybe_alias {
1147+
match self.next_token() {
11781148
// Accept any identifier after `AS` (though many dialects have restrictions on
11791149
// keywords that may appear here). If there's no `AS`: don't parse keywords,
11801150
// which may start a construct allowed in this position, to be parsed as aliases.
@@ -1192,9 +1162,7 @@ impl Parser {
11921162
if after_as {
11931163
return self.expected("an identifier after AS", not_an_ident);
11941164
}
1195-
if not_an_ident.is_some() {
1196-
self.prev_token();
1197-
}
1165+
self.prev_token();
11981166
Ok(None) // no alias found
11991167
}
12001168
}
@@ -1216,9 +1184,7 @@ impl Parser {
12161184
continue;
12171185
}
12181186
_ => {
1219-
if token.is_some() {
1220-
self.prev_token();
1221-
}
1187+
self.prev_token();
12221188
break;
12231189
}
12241190
}
@@ -1774,15 +1740,22 @@ mod tests {
17741740

17751741
#[test]
17761742
fn test_prev_index() {
1777-
let sql = "SELECT version()";
1743+
let sql = "SELECT version";
17781744
all_dialects().run_parser_method(sql, |parser| {
1779-
assert_eq!(parser.prev_token(), None);
1745+
assert_eq!(parser.peek_token(), Some(Token::make_keyword("SELECT")));
1746+
assert_eq!(parser.next_token(), Some(Token::make_keyword("SELECT")));
1747+
parser.prev_token();
17801748
assert_eq!(parser.next_token(), Some(Token::make_keyword("SELECT")));
17811749
assert_eq!(parser.next_token(), Some(Token::make_word("version", None)));
1782-
assert_eq!(parser.prev_token(), Some(Token::make_word("version", None)));
1750+
parser.prev_token();
17831751
assert_eq!(parser.peek_token(), Some(Token::make_word("version", None)));
1784-
assert_eq!(parser.prev_token(), Some(Token::make_keyword("SELECT")));
1785-
assert_eq!(parser.prev_token(), None);
1752+
assert_eq!(parser.next_token(), Some(Token::make_word("version", None)));
1753+
assert_eq!(parser.peek_token(), None);
1754+
parser.prev_token();
1755+
assert_eq!(parser.next_token(), Some(Token::make_word("version", None)));
1756+
assert_eq!(parser.next_token(), None);
1757+
assert_eq!(parser.next_token(), None);
1758+
parser.prev_token();
17861759
});
17871760
}
17881761
}

0 commit comments

Comments
 (0)