Skip to content

Commit d9591cd

Browse files
andygroveandygrove
authored
Merge pull request #46 from zhzy0077/feature/external_table
Thanks @zhzy0077 !
2 parents d1b5668 + d8f824c commit d9591cd

File tree

4 files changed

+238
-53
lines changed

4 files changed

+238
-53
lines changed

‎src/sqlast/mod.rs

+77-1
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,9 @@ pub enum SQLStatement {
249249
name: SQLObjectName,
250250
/// Optional schema
251251
columns: Vec<SQLColumnDef>,
252+
external: bool,
253+
file_format: Option<FileFormat>,
254+
location: Option<String>,
252255
},
253256
/// ALTER TABLE
254257
SQLAlterTable {
@@ -361,7 +364,30 @@ impl ToString for SQLStatement {
361364
query.to_string()
362365
)
363366
}
364-
SQLStatement::SQLCreateTable { name, columns } => format!(
367+
SQLStatement::SQLCreateTable {
368+
name,
369+
columns,
370+
external,
371+
file_format,
372+
location,
373+
} if *external => format!(
374+
"CREATE EXTERNAL TABLE {} ({}) STORED AS {} LOCATION '{}'",
375+
name.to_string(),
376+
columns
377+
.iter()
378+
.map(|c| c.to_string())
379+
.collect::<Vec<String>>()
380+
.join(", "),
381+
file_format.as_ref().map(|f| f.to_string()).unwrap(),
382+
location.as_ref().unwrap()
383+
),
384+
SQLStatement::SQLCreateTable {
385+
name,
386+
columns,
387+
external: _,
388+
file_format: _,
389+
location: _,
390+
} => format!(
365391
"CREATE TABLE {} ({})",
366392
name.to_string(),
367393
columns
@@ -429,3 +455,53 @@ impl ToString for SQLColumnDef {
429455
s
430456
}
431457
}
458+
459+
/// External table's available file format
460+
#[derive(Debug, Clone, PartialEq)]
461+
pub enum FileFormat {
462+
TEXTFILE,
463+
SEQUENCEFILE,
464+
ORC,
465+
PARQUET,
466+
AVRO,
467+
RCFILE,
468+
JSONFILE,
469+
}
470+
471+
impl ToString for FileFormat {
472+
fn to_string(&self) -> String {
473+
use self::FileFormat::*;
474+
match self {
475+
TEXTFILE => "TEXTFILE".to_string(),
476+
SEQUENCEFILE => "SEQUENCEFILE".to_string(),
477+
ORC => "ORC".to_string(),
478+
PARQUET => "PARQUET".to_string(),
479+
AVRO => "AVRO".to_string(),
480+
RCFILE => "RCFILE".to_string(),
481+
JSONFILE => "TEXTFILE".to_string(),
482+
}
483+
}
484+
}
485+
486+
use sqlparser::ParserError;
487+
use std::str::FromStr;
488+
impl FromStr for FileFormat {
489+
type Err = ParserError;
490+
491+
fn from_str(s: &str) -> Result<Self, Self::Err> {
492+
use self::FileFormat::*;
493+
match s {
494+
"TEXTFILE" => Ok(TEXTFILE),
495+
"SEQUENCEFILE" => Ok(SEQUENCEFILE),
496+
"ORC" => Ok(ORC),
497+
"PARQUET" => Ok(PARQUET),
498+
"AVRO" => Ok(AVRO),
499+
"RCFILE" => Ok(RCFILE),
500+
"JSONFILE" => Ok(JSONFILE),
501+
_ => Err(ParserError::ParserError(format!(
502+
"Unexpected file format: {}",
503+
s
504+
))),
505+
}
506+
}
507+
}

‎src/sqlparser.rs

+82-48
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,8 @@ impl Parser {
620620
} else if self.parse_keyword("MATERIALIZED") || self.parse_keyword("VIEW") {
621621
self.prev_token();
622622
self.parse_create_view()
623+
} else if self.parse_keyword("EXTERNAL") {
624+
self.parse_create_external_table()
623625
} else {
624626
parser_err!(format!(
625627
"Unexpected token after CREATE: {:?}",
@@ -628,6 +630,26 @@ impl Parser {
628630
}
629631
}
630632

633+
pub fn parse_create_external_table(&mut self) -> Result<SQLStatement, ParserError> {
634+
self.expect_keyword("TABLE")?;
635+
let table_name = self.parse_object_name()?;
636+
let columns = self.parse_columns()?;
637+
self.expect_keyword("STORED")?;
638+
self.expect_keyword("AS")?;
639+
let file_format = self.parse_identifier()?.parse::<FileFormat>()?;
640+
641+
self.expect_keyword("LOCATION")?;
642+
let location = self.parse_literal_string()?;
643+
644+
Ok(SQLStatement::SQLCreateTable {
645+
name: table_name,
646+
columns,
647+
external: true,
648+
file_format: Some(file_format),
649+
location: Some(location),
650+
})
651+
}
652+
631653
pub fn parse_create_view(&mut self) -> Result<SQLStatement, ParserError> {
632654
let materialized = self.parse_keyword("MATERIALIZED");
633655
self.expect_keyword("VIEW")?;
@@ -650,60 +672,72 @@ impl Parser {
650672
pub fn parse_create_table(&mut self) -> Result<SQLStatement, ParserError> {
651673
let table_name = self.parse_object_name()?;
652674
// parse optional column list (schema)
675+
let columns = self.parse_columns()?;
676+
677+
Ok(SQLStatement::SQLCreateTable {
678+
name: table_name,
679+
columns,
680+
external: false,
681+
file_format: None,
682+
location: None,
683+
})
684+
}
685+
686+
fn parse_columns(&mut self) -> Result<Vec<SQLColumnDef>, ParserError> {
653687
let mut columns = vec![];
654-
if self.consume_token(&Token::LParen) {
655-
loop {
656-
match self.next_token() {
657-
Some(Token::SQLWord(column_name)) => {
658-
let data_type = self.parse_data_type()?;
659-
let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]);
660-
let is_unique = self.parse_keyword("UNIQUE");
661-
let default = if self.parse_keyword("DEFAULT") {
662-
let expr = self.parse_default_expr(0)?;
663-
Some(expr)
664-
} else {
665-
None
666-
};
667-
let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) {
668-
false
669-
} else if self.parse_keyword("NULL") {
670-
true
671-
} else {
672-
true
673-
};
674-
debug!("default: {:?}", default);
675-
676-
columns.push(SQLColumnDef {
677-
name: column_name.as_sql_ident(),
678-
data_type: data_type,
679-
allow_null,
680-
is_primary,
681-
is_unique,
682-
default,
683-
});
684-
match self.next_token() {
685-
Some(Token::Comma) => {}
686-
Some(Token::RParen) => {
687-
break;
688-
}
689-
other => {
690-
return parser_err!(format!(
691-
"Expected ',' or ')' after column definition but found {:?}",
692-
other
693-
));
694-
}
688+
if !self.consume_token(&Token::LParen) {
689+
return Ok(columns);
690+
}
691+
692+
loop {
693+
match self.next_token() {
694+
Some(Token::SQLWord(column_name)) => {
695+
let data_type = self.parse_data_type()?;
696+
let is_primary = self.parse_keywords(vec!["PRIMARY", "KEY"]);
697+
let is_unique = self.parse_keyword("UNIQUE");
698+
let default = if self.parse_keyword("DEFAULT") {
699+
let expr = self.parse_default_expr(0)?;
700+
Some(expr)
701+
} else {
702+
None
703+
};
704+
let allow_null = if self.parse_keywords(vec!["NOT", "NULL"]) {
705+
false
706+
} else if self.parse_keyword("NULL") {
707+
true
708+
} else {
709+
true
710+
};
711+
debug!("default: {:?}", default);
712+
713+
columns.push(SQLColumnDef {
714+
name: column_name.as_sql_ident(),
715+
data_type,
716+
allow_null,
717+
is_primary,
718+
is_unique,
719+
default,
720+
});
721+
match self.next_token() {
722+
Some(Token::Comma) => {}
723+
Some(Token::RParen) => {
724+
break;
725+
}
726+
other => {
727+
return parser_err!(format!(
728+
"Expected ',' or ')' after column definition but found {:?}",
729+
other
730+
));
695731
}
696732
}
697-
unexpected => {
698-
return parser_err!(format!("Expected column name, got {:?}", unexpected));
699-
}
733+
}
734+
unexpected => {
735+
return parser_err!(format!("Expected column name, got {:?}", unexpected));
700736
}
701737
}
702738
}
703-
Ok(SQLStatement::SQLCreateTable {
704-
name: table_name,
705-
columns,
706-
})
739+
740+
Ok(columns)
707741
}
708742

709743
pub fn parse_table_key(&mut self, constraint_name: SQLIdent) -> Result<TableKey, ParserError> {

‎tests/sqlparser_generic.rs

+58-1
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,60 @@ fn parse_create_table() {
434434
lng double)",
435435
);
436436
match ast {
437-
SQLStatement::SQLCreateTable { name, columns } => {
437+
SQLStatement::SQLCreateTable {
438+
name,
439+
columns,
440+
external: _,
441+
file_format: _,
442+
location: _,
443+
} => {
444+
assert_eq!("uk_cities", name.to_string());
445+
assert_eq!(3, columns.len());
446+
447+
let c_name = &columns[0];
448+
assert_eq!("name", c_name.name);
449+
assert_eq!(SQLType::Varchar(Some(100)), c_name.data_type);
450+
assert_eq!(false, c_name.allow_null);
451+
452+
let c_lat = &columns[1];
453+
assert_eq!("lat", c_lat.name);
454+
assert_eq!(SQLType::Double, c_lat.data_type);
455+
assert_eq!(true, c_lat.allow_null);
456+
457+
let c_lng = &columns[2];
458+
assert_eq!("lng", c_lng.name);
459+
assert_eq!(SQLType::Double, c_lng.data_type);
460+
assert_eq!(true, c_lng.allow_null);
461+
}
462+
_ => assert!(false),
463+
}
464+
}
465+
466+
#[test]
467+
fn parse_create_external_table() {
468+
let sql = String::from(
469+
"CREATE EXTERNAL TABLE uk_cities (\
470+
name VARCHAR(100) NOT NULL,\
471+
lat DOUBLE NULL,\
472+
lng DOUBLE NULL)\
473+
STORED AS TEXTFILE LOCATION '/tmp/example.csv",
474+
);
475+
let ast = one_statement_parses_to(
476+
&sql,
477+
"CREATE EXTERNAL TABLE uk_cities (\
478+
name character varying(100) NOT NULL, \
479+
lat double, \
480+
lng double) \
481+
STORED AS TEXTFILE LOCATION '/tmp/example.csv'",
482+
);
483+
match ast {
484+
SQLStatement::SQLCreateTable {
485+
name,
486+
columns,
487+
external,
488+
file_format,
489+
location,
490+
} => {
438491
assert_eq!("uk_cities", name.to_string());
439492
assert_eq!(3, columns.len());
440493

@@ -452,6 +505,10 @@ fn parse_create_table() {
452505
assert_eq!("lng", c_lng.name);
453506
assert_eq!(SQLType::Double, c_lng.data_type);
454507
assert_eq!(true, c_lng.allow_null);
508+
509+
assert!(external);
510+
assert_eq!(FileFormat::TEXTFILE, file_format.unwrap());
511+
assert_eq!("/tmp/example.csv", location.unwrap());
455512
}
456513
_ => assert!(false),
457514
}

‎tests/sqlparser_postgres.rs

+21-3
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,13 @@ fn parse_create_table_with_defaults() {
163163
active integer NOT NULL)",
164164
);
165165
match one_statement_parses_to(&sql, "") {
166-
SQLStatement::SQLCreateTable { name, columns } => {
166+
SQLStatement::SQLCreateTable {
167+
name,
168+
columns,
169+
external: _,
170+
file_format: _,
171+
location: _,
172+
} => {
167173
assert_eq!("public.customer", name.to_string());
168174
assert_eq!(10, columns.len());
169175

@@ -204,7 +210,13 @@ fn parse_create_table_from_pg_dump() {
204210
active integer
205211
)");
206212
match one_statement_parses_to(&sql, "") {
207-
SQLStatement::SQLCreateTable { name, columns } => {
213+
SQLStatement::SQLCreateTable {
214+
name,
215+
columns,
216+
external: _,
217+
file_format: _,
218+
location: _,
219+
} => {
208220
assert_eq!("public.customer", name.to_string());
209221

210222
let c_customer_id = &columns[0];
@@ -261,7 +273,13 @@ fn parse_create_table_with_inherit() {
261273
)",
262274
);
263275
match verified_stmt(&sql) {
264-
SQLStatement::SQLCreateTable { name, columns } => {
276+
SQLStatement::SQLCreateTable {
277+
name,
278+
columns,
279+
external: _,
280+
file_format: _,
281+
location: _,
282+
} => {
265283
assert_eq!("bazaar.settings", name.to_string());
266284

267285
let c_name = &columns[0];

0 commit comments

Comments
 (0)