From 608f1f8ecd016d3ccfd645a363ccf7f6f78dae03 Mon Sep 17 00:00:00 2001 From: Tyler Brinks Date: Fri, 7 Feb 2025 10:20:14 -0700 Subject: [PATCH 1/3] Add support for Varbinary(MAX) (#1714) --- src/ast/data_type.rs | 45 ++++++++++- src/ast/mod.rs | 2 +- src/parser/mod.rs | 22 +++++- tests/sqlparser_common.rs | 4 +- tests/sqlparser_mssql.rs | 155 +++++++++++++++++++++++++++++++++++++- 5 files changed, 220 insertions(+), 8 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index 1f2b6be97..e67801939 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -85,7 +85,7 @@ pub enum DataType { /// /// [standard]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html#binary-string-type /// [MS SQL Server]: https://learn.microsoft.com/pt-br/sql/t-sql/data-types/binary-and-varbinary-transact-sql?view=sql-server-ver16 - Varbinary(Option), + Varbinary(Option), /// Large binary object with optional length e.g. BLOB, BLOB(1000), [standard], [Oracle] /// /// [standard]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html#binary-large-object-string-type @@ -403,9 +403,7 @@ impl fmt::Display for DataType { } DataType::Clob(size) => format_type_with_optional_length(f, "CLOB", size, false), DataType::Binary(size) => format_type_with_optional_length(f, "BINARY", size, false), - DataType::Varbinary(size) => { - format_type_with_optional_length(f, "VARBINARY", size, false) - } + DataType::Varbinary(size) => format_varbinary_type(f, "VARBINARY", size), DataType::Blob(size) => format_type_with_optional_length(f, "BLOB", size, false), DataType::TinyBlob => write!(f, "TINYBLOB"), DataType::MediumBlob => write!(f, "MEDIUMBLOB"), @@ -667,6 +665,19 @@ fn format_character_string_type( Ok(()) } +fn format_varbinary_type( + f: &mut fmt::Formatter, + sql_type: &str, + size: &Option, +) -> fmt::Result{ + + write!(f, "{sql_type}")?; + if let Some(size) = size{ + write!(f, "({size})")?; + } + Ok(()) +} + fn format_datetime_precision_and_tz( f: &mut fmt::Formatter, sql_type: &'static str, @@ -856,6 +867,32 @@ impl fmt::Display for CharLengthUnits { } } +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum BinaryLength { + IntegerLength { + /// Default (if VARYING) + length: u64, + }, + /// VARBINARY(MAX) used in T-SQL (Microsoft SQL Server) + Max, +} + +impl fmt::Display for BinaryLength { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BinaryLength::IntegerLength { length } => { + write!(f, "{}", length)?; + } + BinaryLength::Max => { + write!(f, "MAX")?; + } + } + Ok(()) + } +} + /// Represents the data type of the elements in an array (if any) as well as /// the syntax used to declare the array. /// diff --git a/src/ast/mod.rs b/src/ast/mod.rs index de2bbafc2..a6621b6f3 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -40,7 +40,7 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::tokenizer::Span; pub use self::data_type::{ - ArrayElemTypeDef, CharLengthUnits, CharacterLength, DataType, EnumMember, ExactNumberInfo, + ArrayElemTypeDef, CharLengthUnits, CharacterLength, BinaryLength, DataType, EnumMember, ExactNumberInfo, StructBracketKind, TimezoneInfo, }; pub use self::dcl::{ diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 30124bbc2..5b9640a5e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8766,7 +8766,7 @@ impl<'a> Parser<'a> { } Keyword::CLOB => Ok(DataType::Clob(self.parse_optional_precision()?)), Keyword::BINARY => Ok(DataType::Binary(self.parse_optional_precision()?)), - Keyword::VARBINARY => Ok(DataType::Varbinary(self.parse_optional_precision()?)), + Keyword::VARBINARY => Ok(DataType::Varbinary(self.parse_optional_binary_length()?)), Keyword::BLOB => Ok(DataType::Blob(self.parse_optional_precision()?)), Keyword::TINYBLOB => Ok(DataType::TinyBlob), Keyword::MEDIUMBLOB => Ok(DataType::MediumBlob), @@ -9649,6 +9649,18 @@ impl<'a> Parser<'a> { } } + pub fn parse_optional_binary_length( + &mut self, + ) -> Result, ParserError> { + if self.consume_token(&Token::LParen) { + let binary_length = self.parse_binary_length()?; + self.expect_token(&Token::RParen)?; + Ok(Some(binary_length)) + } else { + Ok(None) + } + } + pub fn parse_character_length(&mut self) -> Result { if self.parse_keyword(Keyword::MAX) { return Ok(CharacterLength::Max); @@ -9664,6 +9676,14 @@ impl<'a> Parser<'a> { Ok(CharacterLength::IntegerLength { length, unit }) } + pub fn parse_binary_length(&mut self) -> Result { + if self.parse_keyword(Keyword::MAX) { + return Ok(BinaryLength::Max); + } + let length = self.parse_literal_uint()?; + Ok(BinaryLength::IntegerLength { length }) + } + pub fn parse_optional_precision_scale( &mut self, ) -> Result<(Option, Option), ParserError> { diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 3a6183a15..19698522f 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -2632,7 +2632,9 @@ fn parse_cast() { &Expr::Cast { kind: CastKind::Cast, expr: Box::new(Expr::Identifier(Ident::new("id"))), - data_type: DataType::Varbinary(Some(50)), + data_type:DataType::Varbinary(Some(BinaryLength::IntegerLength { + length: 50, + })), format: None, }, expr_from_projection(only(&select.projection)) diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 9046e9e74..9153bb425 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -26,7 +26,7 @@ use helpers::attached_token::AttachedToken; use sqlparser::tokenizer::Span; use test_utils::*; -use sqlparser::ast::DataType::{Int, Text}; +use sqlparser::ast::DataType::{Int, Text, Varbinary}; use sqlparser::ast::DeclareAssignment::MsSqlAssignment; use sqlparser::ast::Value::SingleQuotedString; use sqlparser::ast::*; @@ -1793,6 +1793,159 @@ fn parse_mssql_set_session_value() { ms().verified_stmt("SET ANSI_NULLS, ANSI_PADDING ON"); } +#[test] +fn parse_mssql_varbinary_max_length(){ + let sql = "CREATE TABLE example (var_binary_col VARBINARY(MAX))"; + + assert_eq!(ms_and_generic().verified_stmt(sql), + Statement::CreateTable(CreateTable { + or_replace: false, + temporary: false, + external: false, + global: None, + if_not_exists: false, + transient: false, + volatile: false, + with_options: vec![], + name: ObjectName::from(vec![Ident { + value: "example".to_string(), + quote_style: None, + span: Span::empty(), + },],), + columns: vec![ + ColumnDef { + name: Ident { + value: "var_binary_col".to_string(), + quote_style: None, + span: Span::empty(), + }, + data_type: Varbinary(Some(BinaryLength::Max)), + collation: None, + options: vec![], + } + ], + constraints: vec![], + hive_distribution: HiveDistributionStyle::NONE, + hive_formats: Some(HiveFormat { + row_format: None, + serde_properties: None, + storage: None, + location: None, + },), + table_properties: vec![], + file_format: None, + location: None, + query: None, + without_rowid: false, + like: None, + clone: None, + engine: None, + comment: None, + auto_increment_offset: None, + default_charset: None, + collation: None, + on_commit: None, + on_cluster: None, + primary_key: None, + order_by: None, + partition_by: None, + cluster_by: None, + clustered_by: None, + options: None, + strict: false, + iceberg: false, + copy_grants: false, + enable_schema_evolution: None, + change_tracking: None, + data_retention_time_in_days: None, + max_data_extension_time_in_days: None, + default_ddl_collation: None, + with_aggregation_policy: None, + with_row_access_policy: None, + with_tags: None, + base_location: None, + external_volume: None, + catalog: None, + catalog_sync: None, + storage_serialization_policy: None, + })); + + let sql = "CREATE TABLE example (var_binary_col VARBINARY(50))"; + + assert_eq!(ms_and_generic().verified_stmt(sql), + Statement::CreateTable(CreateTable { + or_replace: false, + temporary: false, + external: false, + global: None, + if_not_exists: false, + transient: false, + volatile: false, + with_options: vec![], + name: ObjectName::from(vec![Ident { + value: "example".to_string(), + quote_style: None, + span: Span::empty(), + },],), + columns: vec![ + ColumnDef { + name: Ident { + value: "var_binary_col".to_string(), + quote_style: None, + span: Span::empty(), + }, + data_type: Varbinary(Some(BinaryLength::IntegerLength {length:50})), + collation: None, + options: vec![], + } + ], + constraints: vec![], + hive_distribution: HiveDistributionStyle::NONE, + hive_formats: Some(HiveFormat { + row_format: None, + serde_properties: None, + storage: None, + location: None, + },), + table_properties: vec![], + file_format: None, + location: None, + query: None, + without_rowid: false, + like: None, + clone: None, + engine: None, + comment: None, + auto_increment_offset: None, + default_charset: None, + collation: None, + on_commit: None, + on_cluster: None, + primary_key: None, + order_by: None, + partition_by: None, + cluster_by: None, + clustered_by: None, + options: None, + strict: false, + iceberg: false, + copy_grants: false, + enable_schema_evolution: None, + change_tracking: None, + data_retention_time_in_days: None, + max_data_extension_time_in_days: None, + default_ddl_collation: None, + with_aggregation_policy: None, + with_row_access_policy: None, + with_tags: None, + base_location: None, + external_volume: None, + catalog: None, + catalog_sync: None, + storage_serialization_policy: None, + })); +} + fn ms() -> TestedDialects { TestedDialects::new(vec![Box::new(MsSqlDialect {})]) } From a9150a97f1fc83d2cd8fe0bd41fd51e222aeee5b Mon Sep 17 00:00:00 2001 From: Tyler Brinks Date: Fri, 7 Feb 2025 10:35:34 -0700 Subject: [PATCH 2/3] Add support for Varbinary(MAX) (#1714) --- tests/sqlparser_mssql.rs | 188 +++++++++------------------------------ 1 file changed, 44 insertions(+), 144 deletions(-) diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 9153bb425..e204a8424 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1797,153 +1797,53 @@ fn parse_mssql_set_session_value() { fn parse_mssql_varbinary_max_length(){ let sql = "CREATE TABLE example (var_binary_col VARBINARY(MAX))"; - assert_eq!(ms_and_generic().verified_stmt(sql), - Statement::CreateTable(CreateTable { - or_replace: false, - temporary: false, - external: false, - global: None, - if_not_exists: false, - transient: false, - volatile: false, - with_options: vec![], - name: ObjectName::from(vec![Ident { - value: "example".to_string(), - quote_style: None, - span: Span::empty(), - },],), - columns: vec![ - ColumnDef { - name: Ident { - value: "var_binary_col".to_string(), - quote_style: None, - span: Span::empty(), - }, - data_type: Varbinary(Some(BinaryLength::Max)), - collation: None, - options: vec![], - } - ], - constraints: vec![], - hive_distribution: HiveDistributionStyle::NONE, - hive_formats: Some(HiveFormat { - row_format: None, - serde_properties: None, - storage: None, - location: None, - },), - table_properties: vec![], - file_format: None, - location: None, - query: None, - without_rowid: false, - like: None, - clone: None, - engine: None, - comment: None, - auto_increment_offset: None, - default_charset: None, - collation: None, - on_commit: None, - on_cluster: None, - primary_key: None, - order_by: None, - partition_by: None, - cluster_by: None, - clustered_by: None, - options: None, - strict: false, - iceberg: false, - copy_grants: false, - enable_schema_evolution: None, - change_tracking: None, - data_retention_time_in_days: None, - max_data_extension_time_in_days: None, - default_ddl_collation: None, - with_aggregation_policy: None, - with_row_access_policy: None, - with_tags: None, - base_location: None, - external_volume: None, - catalog: None, - catalog_sync: None, - storage_serialization_policy: None, - })); + match ms_and_generic().verified_stmt(sql) { + Statement::CreateTable(CreateTable { name, columns, .. }) => { + assert_eq!( + name, + ObjectName::from(vec![Ident { + value: "example".to_string(), + quote_style: None, + span: Span::empty(), + }]) + ); + assert_eq!( + columns, + vec![ColumnDef { + name: Ident::new("var_binary_col"), + data_type: Varbinary(Some(BinaryLength::Max)), + collation: None, + options: vec![] + },], + ); + } + _ => unreachable!(), + } let sql = "CREATE TABLE example (var_binary_col VARBINARY(50))"; - assert_eq!(ms_and_generic().verified_stmt(sql), - Statement::CreateTable(CreateTable { - or_replace: false, - temporary: false, - external: false, - global: None, - if_not_exists: false, - transient: false, - volatile: false, - with_options: vec![], - name: ObjectName::from(vec![Ident { - value: "example".to_string(), - quote_style: None, - span: Span::empty(), - },],), - columns: vec![ - ColumnDef { - name: Ident { - value: "var_binary_col".to_string(), - quote_style: None, - span: Span::empty(), - }, - data_type: Varbinary(Some(BinaryLength::IntegerLength {length:50})), - collation: None, - options: vec![], - } - ], - constraints: vec![], - hive_distribution: HiveDistributionStyle::NONE, - hive_formats: Some(HiveFormat { - row_format: None, - serde_properties: None, - storage: None, - location: None, - },), - table_properties: vec![], - file_format: None, - location: None, - query: None, - without_rowid: false, - like: None, - clone: None, - engine: None, - comment: None, - auto_increment_offset: None, - default_charset: None, - collation: None, - on_commit: None, - on_cluster: None, - primary_key: None, - order_by: None, - partition_by: None, - cluster_by: None, - clustered_by: None, - options: None, - strict: false, - iceberg: false, - copy_grants: false, - enable_schema_evolution: None, - change_tracking: None, - data_retention_time_in_days: None, - max_data_extension_time_in_days: None, - default_ddl_collation: None, - with_aggregation_policy: None, - with_row_access_policy: None, - with_tags: None, - base_location: None, - external_volume: None, - catalog: None, - catalog_sync: None, - storage_serialization_policy: None, - })); + match ms_and_generic().verified_stmt(sql) { + Statement::CreateTable(CreateTable { name, columns, .. }) => { + assert_eq!( + name, + ObjectName::from(vec![Ident { + value: "example".to_string(), + quote_style: None, + span: Span::empty(), + }]) + ); + assert_eq!( + columns, + vec![ColumnDef { + name: Ident::new("var_binary_col"), + data_type: Varbinary(Some(BinaryLength::IntegerLength {length:50})), + collation: None, + options: vec![] + },], + ); + } + _ => unreachable!(), + } } fn ms() -> TestedDialects { From a3fbc8bf04378875e9b6ff60413bae3a40207037 Mon Sep 17 00:00:00 2001 From: Tyler Brinks Date: Tue, 11 Feb 2025 07:51:21 -0700 Subject: [PATCH 3/3] Formatting fixes for PR 1714 --- src/ast/data_type.rs | 7 +++---- src/ast/mod.rs | 4 ++-- src/parser/mod.rs | 4 +--- tests/sqlparser_common.rs | 4 +--- tests/sqlparser_mssql.rs | 12 ++++++------ 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index e67801939..95735dedd 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -669,10 +669,9 @@ fn format_varbinary_type( f: &mut fmt::Formatter, sql_type: &str, size: &Option, -) -> fmt::Result{ - +) -> fmt::Result { write!(f, "{sql_type}")?; - if let Some(size) = size{ + if let Some(size) = size { write!(f, "({size})")?; } Ok(()) @@ -882,7 +881,7 @@ pub enum BinaryLength { impl fmt::Display for BinaryLength { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - BinaryLength::IntegerLength { length } => { + BinaryLength::IntegerLength { length } => { write!(f, "{}", length)?; } BinaryLength::Max => { diff --git a/src/ast/mod.rs b/src/ast/mod.rs index a6621b6f3..9e176f71e 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -40,8 +40,8 @@ use sqlparser_derive::{Visit, VisitMut}; use crate::tokenizer::Span; pub use self::data_type::{ - ArrayElemTypeDef, CharLengthUnits, CharacterLength, BinaryLength, DataType, EnumMember, ExactNumberInfo, - StructBracketKind, TimezoneInfo, + ArrayElemTypeDef, BinaryLength, CharLengthUnits, CharacterLength, DataType, EnumMember, + ExactNumberInfo, StructBracketKind, TimezoneInfo, }; pub use self::dcl::{ AlterRoleOperation, ResetConfig, RoleOption, SecondaryRoles, SetConfigValue, Use, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5b9640a5e..bf6e09b69 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -9649,9 +9649,7 @@ impl<'a> Parser<'a> { } } - pub fn parse_optional_binary_length( - &mut self, - ) -> Result, ParserError> { + pub fn parse_optional_binary_length(&mut self) -> Result, ParserError> { if self.consume_token(&Token::LParen) { let binary_length = self.parse_binary_length()?; self.expect_token(&Token::RParen)?; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 19698522f..8829d9d90 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -2632,9 +2632,7 @@ fn parse_cast() { &Expr::Cast { kind: CastKind::Cast, expr: Box::new(Expr::Identifier(Ident::new("id"))), - data_type:DataType::Varbinary(Some(BinaryLength::IntegerLength { - length: 50, - })), + data_type: DataType::Varbinary(Some(BinaryLength::IntegerLength { length: 50 })), format: None, }, expr_from_projection(only(&select.projection)) diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index e204a8424..0fd67dc3d 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1794,7 +1794,7 @@ fn parse_mssql_set_session_value() { } #[test] -fn parse_mssql_varbinary_max_length(){ +fn parse_mssql_varbinary_max_length() { let sql = "CREATE TABLE example (var_binary_col VARBINARY(MAX))"; match ms_and_generic().verified_stmt(sql) { @@ -1827,16 +1827,16 @@ fn parse_mssql_varbinary_max_length(){ assert_eq!( name, ObjectName::from(vec![Ident { - value: "example".to_string(), - quote_style: None, - span: Span::empty(), - }]) + value: "example".to_string(), + quote_style: None, + span: Span::empty(), + }]) ); assert_eq!( columns, vec![ColumnDef { name: Ident::new("var_binary_col"), - data_type: Varbinary(Some(BinaryLength::IntegerLength {length:50})), + data_type: Varbinary(Some(BinaryLength::IntegerLength { length: 50 })), collation: None, options: vec![] },],