Redshift: Fix parsing for quoted numbered columns

src/dialect/mod.rs

@@ -128,14 +128,23 @@ pub trait Dialect: Debug + Any {
         ch == '"' || ch == '`'
     }
 
-    /// Return the character used to quote identifiers.
-    fn identifier_quote_style(&self, _identifier: &str) -> Option<char> {
+    /// Determine if a character starts a potential nested quoted identifier.
+    /// RedShift support old way of quotation with `[` and it can cover even nested quoted identifier.
+    fn is_nested_delimited_identifier_start(&self, _ch: char) -> bool {
+        false
+    }
+
+    /// Determine if nested quoted characters are presented
+    fn nested_delimited_identifier(
+        &self,
+        mut _chars: Peekable<Chars<'_>>,
+    ) -> Option<(char, Option<char>)> {
         None
     }
 
-    /// Determine if quoted characters are proper for identifier
-    fn is_proper_identifier_inside_quotes(&self, mut _chars: Peekable<Chars<'_>>) -> bool {
-        true
+    /// Return the character used to quote identifiers.
+    fn identifier_quote_style(&self, _identifier: &str) -> Option<char> {
+        None
     }
 
     /// Determine if a character is a valid start character for an unquoted identifier
@@ -848,6 +857,17 @@ mod tests {
                 self.0.is_delimited_identifier_start(ch)
             }
 
+            fn is_nested_delimited_identifier_start(&self, ch: char) -> bool {
+                self.0.is_nested_delimited_identifier_start(ch)
+            }
+
+            fn nested_delimited_identifier(
+                &self,
+                chars: std::iter::Peekable<std::str::Chars<'_>>,
+            ) -> Option<(char, Option<char>)> {
+                self.0.nested_delimited_identifier(chars)
+            }
+
             fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
                 self.0.identifier_quote_style(identifier)
             }
@@ -856,13 +876,6 @@ mod tests {
                 self.0.supports_string_literal_backslash_escape()
             }
 
-            fn is_proper_identifier_inside_quotes(
-                &self,
-                chars: std::iter::Peekable<std::str::Chars<'_>>,
-            ) -> bool {
-                self.0.is_proper_identifier_inside_quotes(chars)
-            }
-
             fn supports_filter_during_aggregation(&self) -> bool {
                 self.0.supports_filter_during_aggregation()
             }

src/dialect/redshift.rs

@@ -32,21 +32,36 @@ pub struct RedshiftSqlDialect {}
 // in the Postgres dialect, the query will be parsed as an array, while in the Redshift dialect it will
 // be a json path
 impl Dialect for RedshiftSqlDialect {
-    fn is_delimited_identifier_start(&self, ch: char) -> bool {
-        ch == '"' || ch == '['
+    fn is_nested_delimited_identifier_start(&self, ch: char) -> bool {
+        ch == '['
     }
 
-    /// Determine if quoted characters are proper for identifier
+    /// Determine if quoted characters are looks like special case of quotation begining with `[`.
     /// It's needed to distinguish treating square brackets as quotes from
     /// treating them as json path. If there is identifier then we assume
     /// there is no json path.
-    fn is_proper_identifier_inside_quotes(&self, mut chars: Peekable<Chars<'_>>) -> bool {
+    fn nested_delimited_identifier(
+        &self,
+        mut chars: Peekable<Chars<'_>>,
+    ) -> Option<(char, Option<char>)> {
+        if chars.peek() != Some(&'[') {
+            return None;
+        }
+
         chars.next();
+
         let mut not_white_chars = chars.skip_while(|ch| ch.is_whitespace()).peekable();
+
         if let Some(&ch) = not_white_chars.peek() {
-            return self.is_identifier_start(ch);
+            if ch == '"' {
+                return Some(('[', Some('"')));
+            }
+            if self.is_identifier_start(ch) {
+                return Some(('[', None));
+            }
         }
-        false
+
+        None
     }
 
     fn is_identifier_start(&self, ch: char) -> bool {

src/tokenizer.rs

@@ -1075,25 +1075,56 @@ impl<'a> Tokenizer<'a> {
                     Ok(Some(Token::DoubleQuotedString(s)))
                 }
                 // delimited (quoted) identifier
+                quote_start if self.dialect.is_delimited_identifier_start(ch) => {
+                    let word = self.tokenize_quoted_identifier(quote_start, chars)?;
+                    Ok(Some(Token::make_word(&word, Some(quote_start))))
+                }
+                // special (quoted) identifier
                 quote_start
-                    if self.dialect.is_delimited_identifier_start(ch)
+                    if self
+                        .dialect
+                        .is_nested_delimited_identifier_start(quote_start)
                         && self
                             .dialect
-                            .is_proper_identifier_inside_quotes(chars.peekable.clone()) =>
+                            .nested_delimited_identifier(chars.peekable.clone())
+                            .is_some() =>
                 {
-                    let error_loc = chars.location();
-                    chars.next(); // consume the opening quote
+                    let (quote_start, nested_quote_start) = self
+                        .dialect
+                        .nested_delimited_identifier(chars.peekable.clone())
+                        .unwrap();
+
+                    let Some(nested_quote_start) = nested_quote_start else {
+                        let word = self.tokenize_quoted_identifier(quote_start, chars)?;
+                        return Ok(Some(Token::make_word(&word, Some(quote_start))));
+                    };
+
+                    let mut word = vec![];
                     let quote_end = Word::matching_end_quote(quote_start);
-                    let (s, last_char) = self.parse_quoted_ident(chars, quote_end);
+                    let nested_quote_end = Word::matching_end_quote(nested_quote_start);
+                    let error_loc = chars.location();
 
-                    if last_char == Some(quote_end) {
-                        Ok(Some(Token::make_word(&s, Some(quote_start))))
-                    } else {
-                        self.tokenizer_error(
+                    chars.next(); // skip the first delimiter
+                    word.push(peeking_take_while(chars, |ch| ch.is_whitespace()));
+                    if chars.peek() != Some(&nested_quote_start) {
+                        return self.tokenizer_error(
+                            error_loc,
+                            format!("Expected nested delimiter '{nested_quote_start}' before EOF."),
+                        );
+                    }
+                    word.push(format!("{nested_quote_start}"));
+                    word.push(self.tokenize_quoted_identifier(nested_quote_end, chars)?);
+                    word.push(format!("{nested_quote_end}"));
+                    word.push(peeking_take_while(chars, |ch| ch.is_whitespace()));
+                    if chars.peek() != Some(&quote_end) {
+                        return self.tokenizer_error(
                             error_loc,
                             format!("Expected close delimiter '{quote_end}' before EOF."),
-                        )
+                        );
                     }
+                    chars.next(); // skip close delimiter
+
+                    Ok(Some(Token::make_word(&word.concat(), Some(quote_start))))
                 }
                 // numbers and period
                 '0'..='9' | '.' => {
@@ -1597,6 +1628,27 @@ impl<'a> Tokenizer<'a> {
         s
     }
 
+    /// Read a quoted identifier
+    fn tokenize_quoted_identifier(
+        &self,
+        quote_start: char,
+        chars: &mut State,
+    ) -> Result<String, TokenizerError> {
+        let error_loc = chars.location();
+        chars.next(); // consume the opening quote
+        let quote_end = Word::matching_end_quote(quote_start);
+        let (s, last_char) = self.parse_quoted_ident(chars, quote_end);
+
+        if last_char == Some(quote_end) {
+            Ok(s)
+        } else {
+            self.tokenizer_error(
+                error_loc,
+                format!("Expected close delimiter '{quote_end}' before EOF."),
+            )
+        }
+    }
+
     /// Read a single quoted string, starting with the opening quote.
     fn tokenize_escaped_single_quoted_string(
         &self,

tests/sqlparser_redshift.rs

@@ -279,6 +279,31 @@ fn test_redshift_json_path() {
         },
         expr_from_projection(only(&select.projection))
     );
+
+    let sql = r#"SELECT db1.sc1.tbl1.col1[0]."id" FROM customer_orders_lineitem"#;
+    let select = dialects.verified_only_select(sql);
+    assert_eq!(
+        &Expr::JsonAccess {
+            value: Box::new(Expr::CompoundIdentifier(vec![
+                Ident::new("db1"),
+                Ident::new("sc1"),
+                Ident::new("tbl1"),
+                Ident::new("col1")
+            ])),
+            path: JsonPath {
+                path: vec![
+                    JsonPathElem::Bracket {
+                        key: Expr::Value(Value::Number("0".parse().unwrap(), false))
+                    },
+                    JsonPathElem::Dot {
+                        key: "id".to_string(),
+                        quoted: true,
+                    }
+                ]
+            }
+        },
+        expr_from_projection(only(&select.projection))
+    );
 }
 
 #[test]
@@ -353,3 +378,20 @@ fn test_parse_json_path_from() {
         _ => panic!(),
     }
 }
+
+#[test]
+fn test_parse_select_numbered_columns() {
+    redshift_and_generic().verified_stmt(r#"SELECT 1 AS "1" FROM a"#);
+    // RedShift specific case - quoted identifier inside square bracket
+    redshift().verified_stmt(r#"SELECT 1 AS ["1"] FROM a"#);
+    redshift().verified_stmt(r#"SELECT 1 AS ["[="] FROM a"#);
+    redshift().verified_stmt(r#"SELECT 1 AS ["=]"] FROM a"#);
+    redshift().verified_stmt(r#"SELECT 1 AS ["a[b]"] FROM a"#);
+}
+
+#[test]
+fn test_parse_create_numbered_columns() {
+    redshift_and_generic().verified_stmt(
+        r#"CREATE TABLE test_table_1 ("1" INT, "d" VARCHAR(155), "2" DOUBLE PRECISION)"#,
+    );
+}