Apply patch for dollar placeholder

See apache#1620

Author: hansott

src/dialect/mod.rs

@@ -592,6 +592,12 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if this dialect allows dollar placeholders
+    /// e.g. `SELECT $var` (SQLite)
+    fn supports_dollar_placeholder(&self) -> bool {
+        false
+    }
+
     /// Does the dialect support with clause in create index statement?
     /// e.g. `CREATE INDEX idx ON t WITH (key = value, key2)`
     fn supports_create_index_with_clause(&self) -> bool {

src/dialect/sqlite.rs

@@ -81,4 +81,8 @@ impl Dialect for SQLiteDialect {
     fn supports_asc_desc_in_column_definition(&self) -> bool {
         true
     }
+
+    fn supports_dollar_placeholder(&self) -> bool {
+        true
+    }
 }

src/tokenizer.rs

@@ -1473,7 +1473,8 @@ impl<'a> Tokenizer<'a> {
 
         chars.next();
 
-        if let Some('$') = chars.peek() {
+        // If the dialect does not support dollar-quoted strings, then `$$` is rather a placeholder.
+        if matches!(chars.peek(), Some('$')) && !self.dialect.supports_dollar_placeholder() {
             chars.next();
 
             let mut is_terminated = false;
@@ -1507,10 +1508,14 @@ impl<'a> Tokenizer<'a> {
             };
         } else {
             value.push_str(&peeking_take_while(chars, |ch| {
-                ch.is_alphanumeric() || ch == '_'
+                ch.is_alphanumeric()
+                    || ch == '_'
+                    // Allow $ as a placeholder character if the dialect supports it
+                    || matches!(ch, '$' if self.dialect.supports_dollar_placeholder())
             }));
 
-            if let Some('$') = chars.peek() {
+            // If the dialect does not support dollar-quoted strings, don't look for the end delimiter.
+            if matches!(chars.peek(), Some('$')) && !self.dialect.supports_dollar_placeholder() {
                 chars.next();
 
                 'searching_for_end: loop {
@@ -2080,7 +2085,7 @@ fn take_char_from_hex_digits(
 mod tests {
     use super::*;
     use crate::dialect::{
-        BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect,
+        BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
     };
     use core::fmt::Debug;
 
@@ -2516,6 +2521,30 @@ mod tests {
         );
     }
 
+    #[test]
+    fn tokenize_dollar_placeholder() {
+        let sql = String::from("SELECT $$, $$ABC$$, $ABC$, $ABC");
+        let dialect = SQLiteDialect {};
+        let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
+        assert_eq!(
+            tokens,
+            vec![
+                Token::make_keyword("SELECT"),
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$$".into()),
+                Token::Comma,
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$$ABC$$".into()),
+                Token::Comma,
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$ABC$".into()),
+                Token::Comma,
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$ABC".into()),
+            ]
+        );
+    }
+
     #[test]
     fn tokenize_dollar_quoted_string_untagged() {
         let sql =

tests/sqlparser_sqlite.rs

@@ -570,6 +570,16 @@ fn test_dollar_identifier_as_placeholder() {
         }
         _ => unreachable!(),
     }
+
+    // $$ is a valid placeholder in SQLite
+    match sqlite().verified_expr("id = $$") {
+        Expr::BinaryOp { op, left, right } => {
+            assert_eq!(op, BinaryOperator::Eq);
+            assert_eq!(left, Box::new(Expr::Identifier(Ident::new("id"))));
+            assert_eq!(right, Box::new(Expr::Value(Placeholder("$$".to_string()))));
+        }
+        _ => unreachable!(),
+    }
 }
 
 fn sqlite() -> TestedDialects {