Rename TokenWithLocation to TokenWithSpan, in backwards compatible way (#1562)

alamb · web-flow · commit a134910a362d · 2024-11-30T07:55:21.000-05:00
diff --git a/src/ast/helpers/attached_token.rs b/src/ast/helpers/attached_token.rs
@@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
 use core::fmt::{self, Debug, Formatter};
 use core::hash::{Hash, Hasher};
 
-use crate::tokenizer::{Token, TokenWithLocation};
+use crate::tokenizer::{Token, TokenWithSpan};
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
@@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut};
 #[derive(Clone)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
-pub struct AttachedToken(pub TokenWithLocation);
+pub struct AttachedToken(pub TokenWithSpan);
 
 impl AttachedToken {
     pub fn empty() -> Self {
-        AttachedToken(TokenWithLocation::wrap(Token::EOF))
+        AttachedToken(TokenWithSpan::wrap(Token::EOF))
     }
 }
 
@@ -75,8 +75,8 @@ impl Hash for AttachedToken {
     }
 }
 
-impl From<TokenWithLocation> for AttachedToken {
-    fn from(value: TokenWithLocation) -> Self {
+impl From<TokenWithSpan> for AttachedToken {
+    fn from(value: TokenWithSpan) -> Self {
         AttachedToken(value)
     }
 }
diff --git a/src/ast/query.rs b/src/ast/query.rs
@@ -27,7 +27,7 @@ use sqlparser_derive::{Visit, VisitMut};
 
 use crate::{
     ast::*,
-    tokenizer::{Token, TokenWithLocation},
+    tokenizer::{Token, TokenWithSpan},
 };
 
 /// The most complete variant of a `SELECT` query expression, optionally
@@ -643,7 +643,7 @@ pub struct WildcardAdditionalOptions {
 impl Default for WildcardAdditionalOptions {
     fn default() -> Self {
         Self {
-            wildcard_token: TokenWithLocation::wrap(Token::Mul).into(),
+            wildcard_token: TokenWithSpan::wrap(Token::Mul).into(),
             opt_ilike: None,
             opt_exclude: None,
             opt_except: None,
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
@@ -265,7 +265,7 @@ enum ParserState {
 }
 
 pub struct Parser<'a> {
-    tokens: Vec<TokenWithLocation>,
+    tokens: Vec<TokenWithSpan>,
     /// The index of the first unprocessed token in [`Parser::tokens`].
     index: usize,
     /// The current state of the parser.
@@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Reset this parser to parse the specified token stream
-    pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithLocation>) -> Self {
+    pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithSpan>) -> Self {
         self.tokens = tokens;
         self.index = 0;
         self
@@ -368,9 +368,9 @@ impl<'a> Parser<'a> {
     /// Reset this parser state to parse the specified tokens
     pub fn with_tokens(self, tokens: Vec<Token>) -> Self {
         // Put in dummy locations
-        let tokens_with_locations: Vec<TokenWithLocation> = tokens
+        let tokens_with_locations: Vec<TokenWithSpan> = tokens
             .into_iter()
-            .map(|token| TokenWithLocation {
+            .map(|token| TokenWithSpan {
                 token,
                 span: Span::empty(),
             })
@@ -1147,7 +1147,7 @@ impl<'a> Parser<'a> {
         match self.peek_token().token {
             Token::LParen | Token::Period => {
                 let mut id_parts: Vec<Ident> = vec![w.to_ident(w_span)];
-                let mut ending_wildcard: Option<TokenWithLocation> = None;
+                let mut ending_wildcard: Option<TokenWithSpan> = None;
                 while self.consume_token(&Token::Period) {
                     let next_token = self.next_token();
                     match next_token.token {
@@ -3273,7 +3273,7 @@ impl<'a> Parser<'a> {
 
     /// Return the first non-whitespace token that has not yet been processed
     /// (or None if reached end-of-file)
-    pub fn peek_token(&self) -> TokenWithLocation {
+    pub fn peek_token(&self) -> TokenWithSpan {
         self.peek_nth_token(0)
     }
 
@@ -3308,38 +3308,38 @@ impl<'a> Parser<'a> {
     /// yet been processed.
     ///
     /// See [`Self::peek_token`] for an example.
-    pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithLocation; N] {
+    pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithSpan; N] {
         let mut index = self.index;
         core::array::from_fn(|_| loop {
             let token = self.tokens.get(index);
             index += 1;
-            if let Some(TokenWithLocation {
+            if let Some(TokenWithSpan {
                 token: Token::Whitespace(_),
                 span: _,
             }) = token
             {
                 continue;
             }
-            break token.cloned().unwrap_or(TokenWithLocation {
+            break token.cloned().unwrap_or(TokenWithSpan {
                 token: Token::EOF,
                 span: Span::empty(),
             });
         })
     }
 
     /// Return nth non-whitespace token that has not yet been processed
-    pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation {
+    pub fn peek_nth_token(&self, mut n: usize) -> TokenWithSpan {
         let mut index = self.index;
         loop {
             index += 1;
             match self.tokens.get(index - 1) {
-                Some(TokenWithLocation {
+                Some(TokenWithSpan {
                     token: Token::Whitespace(_),
                     span: _,
                 }) => continue,
                 non_whitespace => {
                     if n == 0 {
-                        return non_whitespace.cloned().unwrap_or(TokenWithLocation {
+                        return non_whitespace.cloned().unwrap_or(TokenWithSpan {
                             token: Token::EOF,
                             span: Span::empty(),
                         });
@@ -3352,16 +3352,16 @@ impl<'a> Parser<'a> {
 
     /// Return the first token, possibly whitespace, that has not yet been processed
     /// (or None if reached end-of-file).
-    pub fn peek_token_no_skip(&self) -> TokenWithLocation {
+    pub fn peek_token_no_skip(&self) -> TokenWithSpan {
         self.peek_nth_token_no_skip(0)
     }
 
     /// Return nth token, possibly whitespace, that has not yet been processed.
-    pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithLocation {
+    pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithSpan {
         self.tokens
             .get(self.index + n)
             .cloned()
-            .unwrap_or(TokenWithLocation {
+            .unwrap_or(TokenWithSpan {
                 token: Token::EOF,
                 span: Span::empty(),
             })
@@ -3378,25 +3378,25 @@ impl<'a> Parser<'a> {
     /// Return the first non-whitespace token that has not yet been processed
     /// (or None if reached end-of-file) and mark it as processed. OK to call
     /// repeatedly after reaching EOF.
-    pub fn next_token(&mut self) -> TokenWithLocation {
+    pub fn next_token(&mut self) -> TokenWithSpan {
         loop {
             self.index += 1;
             match self.tokens.get(self.index - 1) {
-                Some(TokenWithLocation {
+                Some(TokenWithSpan {
                     token: Token::Whitespace(_),
                     span: _,
                 }) => continue,
                 token => {
                     return token
                         .cloned()
-                        .unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF))
+                        .unwrap_or_else(|| TokenWithSpan::wrap(Token::EOF))
                 }
             }
         }
     }
 
     /// Return the first unprocessed token, possibly whitespace.
-    pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> {
+    pub fn next_token_no_skip(&mut self) -> Option<&TokenWithSpan> {
         self.index += 1;
         self.tokens.get(self.index - 1)
     }
@@ -3408,7 +3408,7 @@ impl<'a> Parser<'a> {
         loop {
             assert!(self.index > 0);
             self.index -= 1;
-            if let Some(TokenWithLocation {
+            if let Some(TokenWithSpan {
                 token: Token::Whitespace(_),
                 span: _,
             }) = self.tokens.get(self.index)
@@ -3420,7 +3420,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Report `found` was encountered instead of `expected`
-    pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) -> Result<T, ParserError> {
+    pub fn expected<T>(&self, expected: &str, found: TokenWithSpan) -> Result<T, ParserError> {
         parser_err!(
             format!("Expected: {expected}, found: {found}"),
             found.span.start
@@ -3435,7 +3435,7 @@ impl<'a> Parser<'a> {
     }
 
     #[must_use]
-    pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithLocation> {
+    pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithSpan> {
         match self.peek_token().token {
             Token::Word(w) if expected == w.keyword => Some(self.next_token()),
             _ => None,
@@ -3524,7 +3524,7 @@ impl<'a> Parser<'a> {
 
     /// If the current token is the `expected` keyword, consume the token.
     /// Otherwise, return an error.
-    pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithLocation, ParserError> {
+    pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithSpan, ParserError> {
         if let Some(token) = self.parse_keyword_token(expected) {
             Ok(token)
         } else {
@@ -3568,7 +3568,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Bail out if the current token is not an expected keyword, or consume it if it is
-    pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithLocation, ParserError> {
+    pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithSpan, ParserError> {
         if self.peek_token() == *expected {
             Ok(self.next_token())
         } else {
@@ -4107,7 +4107,7 @@ impl<'a> Parser<'a> {
             Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))),
             _ => self.expected(
                 "JAR, FILE or ARCHIVE, got {:?}",
-                TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
+                TokenWithSpan::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
             ),
         }
     }
@@ -6832,7 +6832,7 @@ impl<'a> Parser<'a> {
                 if let Some(name) = name {
                     return self.expected(
                         "FULLTEXT or SPATIAL option without constraint name",
-                        TokenWithLocation {
+                        TokenWithSpan {
                             token: Token::make_keyword(&name.to_string()),
                             span: next_token.span,
                         },
@@ -7808,15 +7808,15 @@ impl<'a> Parser<'a> {
                     Some('\'') => Ok(Value::SingleQuotedString(w.value)),
                     _ => self.expected(
                         "A value?",
-                        TokenWithLocation {
+                        TokenWithSpan {
                             token: Token::Word(w),
                             span,
                         },
                     )?,
                 },
                 _ => self.expected(
                     "a concrete value",
-                    TokenWithLocation {
+                    TokenWithSpan {
                         token: Token::Word(w),
                         span,
                     },
@@ -7878,7 +7878,7 @@ impl<'a> Parser<'a> {
             }
             unexpected => self.expected(
                 "a value",
-                TokenWithLocation {
+                TokenWithSpan {
                     token: unexpected,
                     span,
                 },
@@ -7927,7 +7927,7 @@ impl<'a> Parser<'a> {
             Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
             unexpected => self.expected(
                 "a string value",
-                TokenWithLocation {
+                TokenWithSpan {
                     token: unexpected,
                     span,
                 },
@@ -8618,7 +8618,7 @@ impl<'a> Parser<'a> {
                         let token = self
                             .next_token_no_skip()
                             .cloned()
-                            .unwrap_or(TokenWithLocation::wrap(Token::EOF));
+                            .unwrap_or(TokenWithSpan::wrap(Token::EOF));
                         requires_whitespace = match token.token {
                             Token::Word(next_word) if next_word.quote_style.is_none() => {
                                 ident.value.push_str(&next_word.value);
@@ -11683,7 +11683,7 @@ impl<'a> Parser<'a> {
     /// If it is not possible to parse it, will return an option.
     pub fn parse_wildcard_additional_options(
         &mut self,
-        wildcard_token: TokenWithLocation,
+        wildcard_token: TokenWithSpan,
     ) -> Result<WildcardAdditionalOptions, ParserError> {
         let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) {
             self.parse_optional_select_item_ilike()?
@@ -12708,7 +12708,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Consume the parser and return its underlying token buffer
-    pub fn into_tokens(self) -> Vec<TokenWithLocation> {
+    pub fn into_tokens(self) -> Vec<TokenWithSpan> {
         self.tokens
     }
 
diff --git a/src/tokenizer.rs b/src/tokenizer.rs

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};`
`19`	`19`	`use core::fmt::{self, Debug, Formatter};`
`20`	`20`	`use core::hash::{Hash, Hasher};`
`21`	`21`
`22`		`-use crate::tokenizer::{Token, TokenWithLocation};`
	`22`	`+use crate::tokenizer::{Token, TokenWithSpan};`
`23`	`23`
`24`	`24`	`#[cfg(feature = "serde")]`
`25`	`25`	`use serde::{Deserialize, Serialize};`
`@@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut};`
`33`	`33`	`#[derive(Clone)]`
`34`	`34`	`#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]`
`35`	`35`	`#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]`
`36`		`-pub struct AttachedToken(pub TokenWithLocation);`
	`36`	`+pub struct AttachedToken(pub TokenWithSpan);`
`37`	`37`
`38`	`38`	`impl AttachedToken {`
`39`	`39`	`pub fn empty() -> Self {`
`40`		`- AttachedToken(TokenWithLocation::wrap(Token::EOF))`
	`40`	`+ AttachedToken(TokenWithSpan::wrap(Token::EOF))`
`41`	`41`	`}`
`42`	`42`	`}`
`43`	`43`
`@@ -75,8 +75,8 @@ impl Hash for AttachedToken {`
`75`	`75`	`}`
`76`	`76`	`}`
`77`	`77`
`78`		`-impl From<TokenWithLocation> for AttachedToken {`
`79`		`- fn from(value: TokenWithLocation) -> Self {`
	`78`	`+impl From<TokenWithSpan> for AttachedToken {`
	`79`	`+ fn from(value: TokenWithSpan) -> Self {`
`80`	`80`	`AttachedToken(value)`
`81`	`81`	`}`
`82`	`82`	`}`