Skip to content

Support Struct datatype parsing for GenericDialect #1241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 30, 2024
Merged

Support Struct datatype parsing for GenericDialect #1241

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1e079dc
support struct for generic dialect
duongcongtoai Apr 28, 2024
ff59e28
fmt
duongcongtoai Apr 28, 2024
3a426d5
chore: revert test
duongcongtoai Apr 30, 2024
4f53d83
add more coverage
duongcongtoai Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/parser/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6576,7 +6576,7 @@ impl<'a> Parser<'a> {
))))
}
}
Keyword::STRUCT if dialect_of!(self is BigQueryDialect) => {
Keyword::STRUCT if dialect_of!(self is BigQueryDialect | GenericDialect) => {
self.prev_token();
let (field_defs, _trailing_bracket) =
self.parse_struct_type_def(Self::parse_big_query_struct_field_def)?;
Expand Down
46 changes: 26 additions & 20 deletions tests/sqlparser_bigquery.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ fn parse_create_table_with_options() {
#[test]
fn parse_nested_data_types() {
let sql = "CREATE TABLE table (x STRUCT<a ARRAY<INT64>, b BYTES(42)>, y ARRAY<STRUCT<INT64>>)";
match bigquery().one_statement_parses_to(sql, sql) {
match bigquery_and_generic().one_statement_parses_to(sql, sql) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also test for generic dialect

Statement::CreateTable { name, columns, .. } => {
assert_eq!(name, ObjectName(vec!["table".into()]));
assert_eq!(
Expand Down Expand Up @@ -395,19 +395,25 @@ fn parse_nested_data_types() {
fn parse_invalid_brackets() {
let sql = "SELECT STRUCT<INT64>>(NULL)";
assert_eq!(
bigquery().parse_sql_statements(sql).unwrap_err(),
bigquery_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError::ParserError("unmatched > in STRUCT literal".to_string())
);

let sql = "SELECT STRUCT<STRUCT<INT64>>>(NULL)";
assert_eq!(
bigquery().parse_sql_statements(sql).unwrap_err(),
bigquery_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError::ParserError("Expected (, found: >".to_string())
);

let sql = "CREATE TABLE table (x STRUCT<STRUCT<INT64>>>)";
assert_eq!(
bigquery().parse_sql_statements(sql).unwrap_err(),
bigquery_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError::ParserError(
"Expected ',' or ')' after column definition, found: >".to_string()
)
Expand Down Expand Up @@ -445,7 +451,7 @@ fn parse_typeless_struct_syntax() {
// typeless struct syntax https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#typeless_struct_syntax
// syntax: STRUCT( expr1 [AS field_name] [, ... ])
let sql = "SELECT STRUCT(1, 2, 3), STRUCT('abc'), STRUCT(1, t.str_col), STRUCT(1 AS a, 'abc' AS b), STRUCT(str_col AS abc)";
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(5, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand Down Expand Up @@ -510,7 +516,7 @@ fn parse_typed_struct_syntax() {
// syntax: STRUCT<[field_name] field_type, ...>( expr1 [, ... ])

let sql = r#"SELECT STRUCT<INT64>(5), STRUCT<x INT64, y STRING>(1, t.str_col), STRUCT<arr ARRAY<FLOAT64>, str STRUCT<BOOL>>(nested_col)"#;
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(3, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand Down Expand Up @@ -582,7 +588,7 @@ fn parse_typed_struct_syntax() {
);

let sql = r#"SELECT STRUCT<x STRUCT, y ARRAY<STRUCT>>(nested_col)"#;
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(1, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand All @@ -607,7 +613,7 @@ fn parse_typed_struct_syntax() {
);

let sql = r#"SELECT STRUCT<BOOL>(true), STRUCT<BYTES(42)>(B'abc')"#;
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(2, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand All @@ -632,12 +638,12 @@ fn parse_typed_struct_syntax() {
expr_from_projection(&select.projection[1])
);

let sql = r#"SELECT STRUCT<DATE>("2011-05-05"), STRUCT<DATETIME>(DATETIME '1999-01-01 01:23:34.45'), STRUCT<FLOAT64>(5.0), STRUCT<INT64>(1)"#;
let select = bigquery().verified_only_select(sql);
let sql = r#"SELECT STRUCT<DATE>('2011-05-05'), STRUCT<DATETIME>(DATETIME '1999-01-01 01:23:34.45'), STRUCT<FLOAT64>(5.0), STRUCT<INT64>(1)"#;
Copy link
Contributor Author

@duongcongtoai duongcongtoai Apr 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

change " to ', so it is also understandable by generic dialect

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rather than updating the existing test, can you please keep the existing test (that uses ") and add new coverage for bigquery_and_generic dialect (with the ')?

This is to avoid losing existing test coverage

Copy link
Contributor Author

@duongcongtoai duongcongtoai Apr 30, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i added extra test, was thinking of using a scenario struct to reuse the query logic and just replacing " with ' in the middle before invoking the parser, but i guess a bit boilerplating is easier to read

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, thank you -- I know it is a lot of repetition but it is easier to understand I think. Thank you 🙏

let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(4, select.projection.len());
assert_eq!(
&Expr::Struct {
values: vec![Expr::Value(Value::DoubleQuotedString(
values: vec![Expr::Value(Value::SingleQuotedString(
"2011-05-05".to_string()
)),],
fields: vec![StructField {
Expand Down Expand Up @@ -682,7 +688,7 @@ fn parse_typed_struct_syntax() {
);

let sql = r#"SELECT STRUCT<INTERVAL>(INTERVAL '1-2 3 4:5:6.789999'), STRUCT<JSON>(JSON '{"class" : {"students" : [{"name" : "Jane"}]}}')"#;
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(2, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand Down Expand Up @@ -716,12 +722,12 @@ fn parse_typed_struct_syntax() {
expr_from_projection(&select.projection[1])
);

let sql = r#"SELECT STRUCT<STRING(42)>("foo"), STRUCT<TIMESTAMP>(TIMESTAMP '2008-12-25 15:30:00 America/Los_Angeles'), STRUCT<TIME>(TIME '15:30:00')"#;
let select = bigquery().verified_only_select(sql);
let sql = r#"SELECT STRUCT<STRING(42)>('foo'), STRUCT<TIMESTAMP>(TIMESTAMP '2008-12-25 15:30:00 America/Los_Angeles'), STRUCT<TIME>(TIME '15:30:00')"#;
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(3, select.projection.len());
assert_eq!(
&Expr::Struct {
values: vec![Expr::Value(Value::DoubleQuotedString("foo".to_string())),],
values: vec![Expr::Value(Value::SingleQuotedString("foo".to_string())),],
fields: vec![StructField {
field_name: None,
field_type: DataType::String(Some(42))
Expand Down Expand Up @@ -758,7 +764,7 @@ fn parse_typed_struct_syntax() {
);

let sql = r#"SELECT STRUCT<NUMERIC>(NUMERIC '1'), STRUCT<BIGNUMERIC>(BIGNUMERIC '1')"#;
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(2, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand Down Expand Up @@ -790,8 +796,8 @@ fn parse_typed_struct_syntax() {

#[test]
fn parse_typed_struct_with_field_name() {
let sql = r#"SELECT STRUCT<x INT64>(5), STRUCT<y STRING>("foo")"#;
let select = bigquery().verified_only_select(sql);
let sql = r#"SELECT STRUCT<x INT64>(5), STRUCT<y STRING>('foo')"#;
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(2, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand All @@ -805,7 +811,7 @@ fn parse_typed_struct_with_field_name() {
);
assert_eq!(
&Expr::Struct {
values: vec![Expr::Value(Value::DoubleQuotedString("foo".to_string())),],
values: vec![Expr::Value(Value::SingleQuotedString("foo".to_string())),],
fields: vec![StructField {
field_name: Some(Ident::from("y")),
field_type: DataType::String(None)
Expand All @@ -815,7 +821,7 @@ fn parse_typed_struct_with_field_name() {
);

let sql = r#"SELECT STRUCT<x INT64, y INT64>(5, 5)"#;
let select = bigquery().verified_only_select(sql);
let select = bigquery_and_generic().verified_only_select(sql);
assert_eq!(1, select.projection.len());
assert_eq!(
&Expr::Struct {
Expand Down
Loading