feat: support byte string literal in bq

Author: togami2864

src/ast/value.rs

@@ -41,6 +41,10 @@ pub enum Value {
     /// See [Postgres docs](https://www.postgresql.org/docs/8.3/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS)
     /// for more details.
     EscapedStringLiteral(String),
+    /// B'string value'
+    SingleQuotedByteStringLiteral(String),
+    /// B"string value"
+    DoubleQuotedByteStringLiteral(String),
     /// N'string value'
     NationalStringLiteral(String),
     /// X'hex value'
@@ -65,6 +69,8 @@ impl fmt::Display for Value {
             Value::SingleQuotedString(v) => write!(f, "'{}'", escape_single_quote_string(v)),
             Value::DollarQuotedString(v) => write!(f, "{}", v),
             Value::EscapedStringLiteral(v) => write!(f, "E'{}'", escape_escaped_string(v)),
+            Value::SingleQuotedByteStringLiteral(v) => write!(f, "B'{}'", v),
+            Value::DoubleQuotedByteStringLiteral(v) => write!(f, "B\"{}\"", v),
             Value::NationalStringLiteral(v) => write!(f, "N'{}'", v),
             Value::HexStringLiteral(v) => write!(f, "X'{}'", v),
             Value::Boolean(v) => write!(f, "{}", v),

src/parser.rs

@@ -779,6 +779,8 @@ impl<'a> Parser<'a> {
             | Token::SingleQuotedString(_)
             | Token::DoubleQuotedString(_)
             | Token::DollarQuotedString(_)
+            | Token::SingleQuotedByteStringLiteral(_)
+            | Token::DoubleQuotedByteStringLiteral(_)
             | Token::NationalStringLiteral(_)
             | Token::HexStringLiteral(_) => {
                 self.prev_token();
@@ -4112,6 +4114,12 @@ impl<'a> Parser<'a> {
             Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())),
             Token::DoubleQuotedString(ref s) => Ok(Value::DoubleQuotedString(s.to_string())),
             Token::DollarQuotedString(ref s) => Ok(Value::DollarQuotedString(s.clone())),
+            Token::SingleQuotedByteStringLiteral(ref s) => {
+                Ok(Value::SingleQuotedByteStringLiteral(s.clone()))
+            }
+            Token::DoubleQuotedByteStringLiteral(ref s) => {
+                Ok(Value::DoubleQuotedByteStringLiteral(s.clone()))
+            }
             Token::NationalStringLiteral(ref s) => Ok(Value::NationalStringLiteral(s.to_string())),
             Token::EscapedStringLiteral(ref s) => Ok(Value::EscapedStringLiteral(s.to_string())),
             Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),

src/tokenizer.rs

@@ -58,6 +58,10 @@ pub enum Token {
     DoubleQuotedString(String),
     /// Dollar quoted string: i.e: $$string$$ or $tag_name$string$tag_name$
     DollarQuotedString(DollarQuotedString),
+    /// Byte string literal: i.e: b'string' or B'string'
+    SingleQuotedByteStringLiteral(String),
+    /// Byte string literal: i.e: b"string" or B"string"
+    DoubleQuotedByteStringLiteral(String),
     /// "National" string literal: i.e: N'string'
     NationalStringLiteral(String),
     /// "escaped" string literal, which are an extension to the SQL standard: i.e: e'first \n second' or E 'first \n second'
@@ -186,6 +190,8 @@ impl fmt::Display for Token {
             Token::SingleQuotedString(ref s) => write!(f, "'{}'", s),
             Token::DoubleQuotedString(ref s) => write!(f, "\"{}\"", s),
             Token::DollarQuotedString(ref s) => write!(f, "{}", s),
+            Token::SingleQuotedByteStringLiteral(ref s) => write!(f, "B'{}'", s),
+            Token::DoubleQuotedByteStringLiteral(ref s) => write!(f, "B\"{}\"", s),
             Token::NationalStringLiteral(ref s) => write!(f, "N'{}'", s),
             Token::EscapedStringLiteral(ref s) => write!(f, "E'{}'", s),
             Token::HexStringLiteral(ref s) => write!(f, "X'{}'", s),
@@ -493,6 +499,25 @@ impl<'a> Tokenizer<'a> {
                     }
                     Ok(Some(Token::Whitespace(Whitespace::Newline)))
                 }
+                // BigQuery uses b or B for byte string literal
+                b @ 'B' | b @ 'b' => {
+                    chars.next(); // consume
+                    match chars.peek() {
+                        Some('\'') => {
+                            let s = self.tokenize_quoted_string(chars, '\'')?;
+                            Ok(Some(Token::SingleQuotedByteStringLiteral(s)))
+                        }
+                        Some('\"') => {
+                            let s = self.tokenize_quoted_string(chars, '\"')?;
+                            Ok(Some(Token::DoubleQuotedByteStringLiteral(s)))
+                        }
+                        _ => {
+                            // regular identifier starting with an "b" or "B"
+                            let s = self.tokenize_word(b, chars);
+                            Ok(Some(Token::make_word(&s, None)))
+                        }
+                    }
+                }
                 // Redshift uses lower case n for national string literal
                 n @ 'N' | n @ 'n' => {
                     chars.next(); // consume, to check the next char

tests/sqlparser_bigquery.rs

@@ -33,6 +33,21 @@ fn parse_literal_string() {
     );
 }
 
+#[test]
+fn parse_byte_literal() {
+    let sql = r#"SELECT B'abc', B"abc""#;
+    let select = bigquery().verified_only_select(sql);
+    assert_eq!(2, select.projection.len());
+    assert_eq!(
+        &Expr::Value(Value::SingleQuotedByteStringLiteral("abc".to_string())),
+        expr_from_projection(&select.projection[0])
+    );
+    assert_eq!(
+        &Expr::Value(Value::DoubleQuotedByteStringLiteral("abc".to_string())),
+        expr_from_projection(&select.projection[1])
+    );
+}
+
 #[test]
 fn parse_table_identifiers() {
     fn test_table_ident(ident: &str, expected: Vec<Ident>) {