Support ?-based jsonb operators in Postgres

We already support most postgres jsonb operators, but were missing the
?-based ones. This commit adds support for them in both the tokenizer
and the AST enums.  This is simplified in the tokenizer with a
dialect-specific carve-out, since Postgres thankfully does not also use
? for anonymous prepared statement parameters.

Author: ReppCodes

src/ast/mod.rs

@@ -277,28 +277,34 @@ impl fmt::Display for Interval {
 pub enum JsonOperator {
     /// -> keeps the value as json
     Arrow,
-    /// ->> keeps the value as text or int.
-    LongArrow,
+    /// jsonb <@ jsonb -> boolean: Test whether right json contains the left json
+    ArrowAt,
+    /// jsonb @> jsonb -> boolean: Test whether left json contains the right json
+    AtArrow,
+    /// jsonb @@ jsonpath → boolean: Returns the result of a JSON path predicate check
+    /// for the specified JSON value. Only the first item of the result is taken into
+    /// account. If the result is not Boolean, then NULL is returned.
+    AtAt,
+    /// jsonb @? jsonpath -> boolean: Does JSON path return any item for the specified
+    /// JSON value?
+    AtQuestion,
+    /// : Colon is used by Snowflake (Which is similar to LongArrow)
+    Colon,
     /// #> Extracts JSON sub-object at the specified path
     HashArrow,
     /// #>> Extracts JSON sub-object at the specified path as text
     HashLongArrow,
-    /// : Colon is used by Snowflake (Which is similar to LongArrow)
-    Colon,
-    /// jsonb @> jsonb -> boolean: Test whether left json contains the right json
-    AtArrow,
-    /// jsonb <@ jsonb -> boolean: Test whether right json contains the left json
-    ArrowAt,
     /// jsonb #- text[] -> jsonb: Deletes the field or array element at the specified
     /// path, where path elements can be either field keys or array indexes.
     HashMinus,
-    /// jsonb @? jsonpath -> boolean: Does JSON path return any item for the specified
-    /// JSON value?
-    AtQuestion,
-    /// jsonb @@ jsonpath → boolean: Returns the result of a JSON path predicate check
-    /// for the specified JSON value. Only the first item of the result is taken into
-    /// account. If the result is not Boolean, then NULL is returned.
-    AtAt,
+    /// ->> keeps the value as text or int.
+    LongArrow,
+    /// jsonb ? string: Does the string exist as a top-level key within the JSON value
+    Question,
+    /// jsonb ?& text[]: Do all of these array strings exist as top-level keys?
+    QuestionAnd,
+    /// jsonb ?| text[]: Do any of these array strings exist as top-level keys?
+    QuestionPipe,
 }
 
 impl fmt::Display for JsonOperator {
@@ -307,25 +313,30 @@ impl fmt::Display for JsonOperator {
             JsonOperator::Arrow => {
                 write!(f, "->")
             }
-            JsonOperator::LongArrow => {
-                write!(f, "->>")
+            JsonOperator::ArrowAt => write!(f, "<@"),
+            JsonOperator::AtArrow => {
+                write!(f, "@>")
+            }
+            JsonOperator::AtAt => write!(f, "@@"),
+            JsonOperator::AtQuestion => write!(f, "@?"),
+            JsonOperator::Colon => {
+                write!(f, ":")
             }
             JsonOperator::HashArrow => {
                 write!(f, "#>")
             }
             JsonOperator::HashLongArrow => {
                 write!(f, "#>>")
             }
-            JsonOperator::Colon => {
-                write!(f, ":")
+            JsonOperator::HashMinus => write!(f, "#-"),
+            JsonOperator::LongArrow => {
+                write!(f, "->>")
             }
-            JsonOperator::AtArrow => {
-                write!(f, "@>")
+            JsonOperator::Question => {
+                write!(f, "?")
             }
-            JsonOperator::ArrowAt => write!(f, "<@"),
-            JsonOperator::HashMinus => write!(f, "#-"),
-            JsonOperator::AtQuestion => write!(f, "@?"),
-            JsonOperator::AtAt => write!(f, "@@"),
+            JsonOperator::QuestionAnd => write!(f, "?&"),
+            JsonOperator::QuestionPipe => write!(f, "?|"),
         }
     }
 }

src/parser/mod.rs

@@ -2524,25 +2524,31 @@ impl<'a> Parser<'a> {
                 right: Box::new(Expr::Value(self.parse_value()?)),
             })
         } else if Token::Arrow == tok
-            || Token::LongArrow == tok
+            || Token::ArrowAt == tok
+            || Token::AtArrow == tok
+            || Token::AtAt == tok
+            || Token::AtQuestion == tok
             || Token::HashArrow == tok
             || Token::HashLongArrow == tok
-            || Token::AtArrow == tok
-            || Token::ArrowAt == tok
             || Token::HashMinus == tok
-            || Token::AtQuestion == tok
-            || Token::AtAt == tok
+            || Token::LongArrow == tok
+            || Token::Question == tok
+            || Token::QuestionAnd == tok
+            || Token::QuestionPipe == tok
         {
             let operator = match tok.token {
                 Token::Arrow => JsonOperator::Arrow,
-                Token::LongArrow => JsonOperator::LongArrow,
+                Token::ArrowAt => JsonOperator::ArrowAt,
+                Token::AtArrow => JsonOperator::AtArrow,
+                Token::AtAt => JsonOperator::AtAt,
+                Token::AtQuestion => JsonOperator::AtQuestion,
                 Token::HashArrow => JsonOperator::HashArrow,
                 Token::HashLongArrow => JsonOperator::HashLongArrow,
-                Token::AtArrow => JsonOperator::AtArrow,
-                Token::ArrowAt => JsonOperator::ArrowAt,
                 Token::HashMinus => JsonOperator::HashMinus,
-                Token::AtQuestion => JsonOperator::AtQuestion,
-                Token::AtAt => JsonOperator::AtAt,
+                Token::LongArrow => JsonOperator::LongArrow,
+                Token::Question => JsonOperator::Question,
+                Token::QuestionAnd => JsonOperator::QuestionAnd,
+                Token::QuestionPipe => JsonOperator::QuestionPipe,
                 _ => unreachable!(),
             };
             Ok(Expr::JsonAccess {
@@ -2788,7 +2794,10 @@ impl<'a> Parser<'a> {
             | Token::ArrowAt
             | Token::HashMinus
             | Token::AtQuestion
-            | Token::AtAt => Ok(50),
+            | Token::AtAt
+            | Token::Question
+            | Token::QuestionAnd
+            | Token::QuestionPipe => Ok(50),
             _ => Ok(0),
         }
     }

src/tokenizer.rs

@@ -36,7 +36,8 @@ use sqlparser_derive::{Visit, VisitMut};
 
 use crate::ast::DollarQuotedString;
 use crate::dialect::{
-    BigQueryDialect, DuckDbDialect, GenericDialect, HiveDialect, SnowflakeDialect,
+    BigQueryDialect, DuckDbDialect, GenericDialect, HiveDialect, PostgreSqlDialect,
+    SnowflakeDialect,
 };
 use crate::dialect::{Dialect, MySqlDialect};
 use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
@@ -199,6 +200,15 @@ pub enum Token {
     /// for the specified JSON value. Only the first item of the result is taken into
     /// account. If the result is not Boolean, then NULL is returned.
     AtAt,
+    /// jsonb ? text -> boolean: Checks whether the string exists as a top-level key within the
+    /// jsonb object
+    Question,
+    /// jsonb ?& text[] -> boolean: Check whether all members of the text array exist as top-level
+    /// keys within the jsonb object
+    QuestionAnd,
+    /// jsonb ?| text[] -> boolean: Check whether any member of the text array exists as top-level
+    /// keys within the jsonb object
+    QuestionPipe,
 }
 
 impl fmt::Display for Token {
@@ -278,6 +288,9 @@ impl fmt::Display for Token {
             Token::HashMinus => write!(f, "#-"),
             Token::AtQuestion => write!(f, "@?"),
             Token::AtAt => write!(f, "@@"),
+            Token::Question => write!(f, "?"),
+            Token::QuestionAnd => write!(f, "?&"),
+            Token::QuestionPipe => write!(f, "?|"),
         }
     }
 }
@@ -1059,6 +1072,15 @@ impl<'a> Tokenizer<'a> {
                         _ => Ok(Some(Token::AtSign)),
                     }
                 }
+                // Postgres uses ? for jsonb operators, not prepared statements
+                '?' if dialect_of!(self is PostgreSqlDialect) => {
+                    chars.next();
+                    match chars.peek() {
+                        Some('|') => self.consume_and_return(chars, Token::QuestionPipe),
+                        Some('&') => self.consume_and_return(chars, Token::QuestionAnd),
+                        _ => self.consume_and_return(chars, Token::Question),
+                    }
+                }
                 '?' => {
                     chars.next();
                     let s = peeking_take_while(chars, |ch| ch.is_numeric());

tests/sqlparser_common.rs

@@ -7655,17 +7655,6 @@ fn test_lock_nonblock() {
 
 #[test]
 fn test_placeholder() {
-    let sql = "SELECT * FROM student WHERE id = ?";
-    let ast = verified_only_select(sql);
-    assert_eq!(
-        ast.selection,
-        Some(Expr::BinaryOp {
-            left: Box::new(Expr::Identifier(Ident::new("id"))),
-            op: BinaryOperator::Eq,
-            right: Box::new(Expr::Value(Value::Placeholder("?".into()))),
-        })
-    );
-
     let dialects = TestedDialects {
         dialects: vec![
             Box::new(GenericDialect {}),
@@ -7705,6 +7694,32 @@ fn test_placeholder() {
         }),
     );
 
+    let dialects = TestedDialects {
+        dialects: vec![
+            Box::new(GenericDialect {}),
+            Box::new(DuckDbDialect {}),
+            // Note: `?` is for jsonb operators in PostgreSqlDialect
+            // Box::new(PostgreSqlDialect {}),
+            Box::new(MsSqlDialect {}),
+            Box::new(AnsiDialect {}),
+            Box::new(BigQueryDialect {}),
+            Box::new(SnowflakeDialect {}),
+            // Note: `$` is the starting word for the HiveDialect identifier
+            // Box::new(sqlparser::dialect::HiveDialect {}),
+        ],
+        options: None,
+    };
+    let sql = "SELECT * FROM student WHERE id = ?";
+    let ast = dialects.verified_only_select(sql);
+    assert_eq!(
+        ast.selection,
+        Some(Expr::BinaryOp {
+            left: Box::new(Expr::Identifier(Ident::new("id"))),
+            op: BinaryOperator::Eq,
+            right: Box::new(Expr::Value(Value::Placeholder("?".into()))),
+        })
+    );
+
     let sql = "SELECT $fromage_français, :x, ?123";
     let ast = dialects.verified_only_select(sql);
     assert_eq!(

tests/sqlparser_postgres.rs

@@ -2358,6 +2358,51 @@ fn test_json() {
         },
         select.selection.unwrap(),
     );
+
+    let sql = r#"SELECT info FROM orders WHERE info ? 'b'"#;
+    let select = pg().verified_only_select(sql);
+    assert_eq!(
+        Expr::JsonAccess {
+            left: Box::new(Expr::Identifier(Ident::new("info"))),
+            operator: JsonOperator::Question,
+            right: Box::new(Expr::Value(Value::SingleQuotedString("b".to_string()))),
+        },
+        select.selection.unwrap(),
+    );
+
+    let sql = r#"SELECT info FROM orders WHERE info ?& ARRAY['b', 'c']"#;
+    let select = pg().verified_only_select(sql);
+    assert_eq!(
+        Expr::JsonAccess {
+            left: Box::new(Expr::Identifier(Ident::new("info"))),
+            operator: JsonOperator::QuestionAnd,
+            right: Box::new(Expr::Array(Array {
+                elem: vec![
+                    Expr::Value(Value::SingleQuotedString("b".to_string())),
+                    Expr::Value(Value::SingleQuotedString("c".to_string()))
+                ],
+                named: true
+            }))
+        },
+        select.selection.unwrap(),
+    );
+
+    let sql = r#"SELECT info FROM orders WHERE info ?| ARRAY['b', 'c']"#;
+    let select = pg().verified_only_select(sql);
+    assert_eq!(
+        Expr::JsonAccess {
+            left: Box::new(Expr::Identifier(Ident::new("info"))),
+            operator: JsonOperator::QuestionPipe,
+            right: Box::new(Expr::Array(Array {
+                elem: vec![
+                    Expr::Value(Value::SingleQuotedString("b".to_string())),
+                    Expr::Value(Value::SingleQuotedString("c".to_string()))
+                ],
+                named: true
+            }))
+        },
+        select.selection.unwrap(),
+    );
 }
 
 #[test]