Only support escape literals for Postgres, Redshift and generic dialect

src/dialect/generic.rs

@@ -139,4 +139,8 @@ impl Dialect for GenericDialect {
     fn supports_user_host_grantee(&self) -> bool {
         true
     }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
 }

src/dialect/mod.rs

@@ -820,6 +820,13 @@ pub trait Dialect: Debug + Any {
     fn supports_set_stmt_without_operator(&self) -> bool {
         false
     }
+
+    /// Returns true if this dialect supports the E'...' syntax for string literals
+    ///
+    /// Postgres: <https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS-ESCAPE>
+    fn supports_string_escape_constant(&self) -> bool {
+        false
+    }
 }
 
 /// This represents the operators for which precedence must be defined

src/dialect/postgresql.rs

@@ -245,6 +245,10 @@ impl Dialect for PostgreSqlDialect {
     fn supports_nested_comments(&self) -> bool {
         true
     }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
 }
 
 pub fn parse_create(parser: &mut Parser) -> Option<Result<Statement, ParserError>> {

src/dialect/redshift.rs

@@ -109,4 +109,8 @@ impl Dialect for RedshiftSqlDialect {
     fn supports_partiql(&self) -> bool {
         true
     }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
 }

src/test_utils.rs

@@ -33,7 +33,7 @@ use core::fmt::Debug;
 
 use crate::dialect::*;
 use crate::parser::{Parser, ParserError};
-use crate::tokenizer::Tokenizer;
+use crate::tokenizer::{Token, Tokenizer};
 use crate::{ast::*, parser::ParserOptions};
 
 #[cfg(test)]
@@ -237,6 +237,22 @@ impl TestedDialects {
     pub fn verified_expr(&self, sql: &str) -> Expr {
         self.expr_parses_to(sql, sql)
     }
+
+    /// Check that the tokenizer returns the expected tokens for the given SQL.
+    pub fn tokenizes_to(&self, sql: &str, expected: Vec<Token>) {
+        if self.dialects.len() == 0 {
+            panic!("No dialects to test");
+        }
+
+        self.dialects.iter().for_each(|dialect| {
+            let mut tokenizer = Tokenizer::new(&**dialect, sql);
+            if let Some(options) = &self.options {
+                tokenizer = tokenizer.with_unescape(options.unescape);
+            }
+            let tokens = tokenizer.tokenize().unwrap();
+            assert_eq!(expected, tokens, "Tokenized differently for {:?}", dialect);
+        });
+    }
 }
 
 /// Returns all available dialects.

src/tokenizer.rs

@@ -982,7 +982,7 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 // PostgreSQL accepts "escape" string constants, which are an extension to the SQL standard.
-                x @ 'e' | x @ 'E' => {
+                x @ 'e' | x @ 'E' if self.dialect.supports_string_escape_constant() => {
                     let starting_loc = chars.location();
                     chars.next(); // consume, to check the next char
                     match chars.peek() {
@@ -2155,6 +2155,7 @@ mod tests {
     use crate::dialect::{
         BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
     };
+    use crate::test_utils::all_dialects_where;
     use core::fmt::Debug;
 
     #[test]
@@ -3543,4 +3544,64 @@ mod tests {
         ];
         compare(expected, tokens);
     }
+
+    #[test]
+    fn test_string_escape_constant_not_supported() {
+        all_dialects_where(|dialect| {
+            !dialect.supports_string_escape_constant()
+                && !dialect.supports_string_literal_backslash_escape()
+        })
+        .tokenizes_to(
+            "select e'\\n'",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::make_word("e", None),
+                Token::SingleQuotedString("\\n".to_string()),
+            ],
+        );
+
+        all_dialects_where(|dialect| {
+            !dialect.supports_string_escape_constant()
+                && !dialect.supports_string_literal_backslash_escape()
+        })
+        .tokenizes_to(
+            "select E'\\n'",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::make_word("E", None),
+                Token::SingleQuotedString("\\n".to_string()),
+            ],
+        );
+    }
+
+    #[test]
+    fn test_string_escape_constant_supported() {
+        all_dialects_where(|dialect| {
+            dialect.supports_string_escape_constant()
+                && !dialect.supports_string_literal_backslash_escape()
+        })
+        .tokenizes_to(
+            "select e'\\''",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::EscapedStringLiteral("'".to_string()),
+            ],
+        );
+
+        all_dialects_where(|dialect| {
+            dialect.supports_string_escape_constant()
+                && !dialect.supports_string_literal_backslash_escape()
+        })
+        .tokenizes_to(
+            "select E'\\''",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::EscapedStringLiteral("'".to_string()),
+            ],
+        );
+    }
 }