@@ -541,6 +541,7 @@ impl<'a> Tokenizer<'a> {
541
541
chars. next ( ) ; // consume the first char
542
542
let s = self . tokenize_word ( ch, chars) ;
543
543
544
+ // TODO: implement parsing of exponent here
544
545
if s. chars ( ) . all ( |x| ( '0' ..='9' ) . contains ( & x) || x == '.' ) {
545
546
let mut inner_state = State {
546
547
peekable : s. chars ( ) . peekable ( ) ,
@@ -617,6 +618,36 @@ impl<'a> Tokenizer<'a> {
617
618
return Ok ( Some ( Token :: Period ) ) ;
618
619
}
619
620
621
+ // Parse exponent as number
622
+ if chars. peek ( ) == Some ( & 'e' ) || chars. peek ( ) == Some ( & 'E' ) {
623
+ let mut char_clone = chars. peekable . clone ( ) ;
624
+ let mut exponent_part = String :: new ( ) ;
625
+ exponent_part. push ( char_clone. next ( ) . unwrap ( ) ) ;
626
+
627
+ // Optional sign
628
+ match char_clone. peek ( ) {
629
+ Some ( & c) if matches ! ( c, '+' | '-' ) => {
630
+ exponent_part. push ( c) ;
631
+ char_clone. next ( ) ;
632
+ }
633
+ _ => ( ) ,
634
+ }
635
+
636
+ match char_clone. peek ( ) {
637
+ // Definitely an exponent, get original iterator up to speed and use it
638
+ Some ( & c) if matches ! ( c, '0' ..='9' ) => {
639
+ for _ in 0 ..exponent_part. len ( ) {
640
+ chars. next ( ) ;
641
+ }
642
+ exponent_part +=
643
+ & peeking_take_while ( chars, |ch| matches ! ( ch, '0' ..='9' ) ) ;
644
+ s += exponent_part. as_str ( ) ;
645
+ }
646
+ // Not an exponent, discard the work done
647
+ _ => ( ) ,
648
+ }
649
+ }
650
+
620
651
let long = if chars. peek ( ) == Some ( & 'L' ) {
621
652
chars. next ( ) ;
622
653
true
@@ -1091,6 +1122,41 @@ mod tests {
1091
1122
compare ( expected, tokens) ;
1092
1123
}
1093
1124
1125
+ #[ test]
1126
+ fn tokenize_select_exponent ( ) {
1127
+ let sql = String :: from ( "SELECT 1e10, 1e-10, 1e+10, 1ea, 1e-10a, 1e-10-10" ) ;
1128
+ let dialect = GenericDialect { } ;
1129
+ let mut tokenizer = Tokenizer :: new ( & dialect, & sql) ;
1130
+ let tokens = tokenizer. tokenize ( ) . unwrap ( ) ;
1131
+
1132
+ let expected = vec ! [
1133
+ Token :: make_keyword( "SELECT" ) ,
1134
+ Token :: Whitespace ( Whitespace :: Space ) ,
1135
+ Token :: Number ( String :: from( "1e10" ) , false ) ,
1136
+ Token :: Comma ,
1137
+ Token :: Whitespace ( Whitespace :: Space ) ,
1138
+ Token :: Number ( String :: from( "1e-10" ) , false ) ,
1139
+ Token :: Comma ,
1140
+ Token :: Whitespace ( Whitespace :: Space ) ,
1141
+ Token :: Number ( String :: from( "1e+10" ) , false ) ,
1142
+ Token :: Comma ,
1143
+ Token :: Whitespace ( Whitespace :: Space ) ,
1144
+ Token :: Number ( String :: from( "1" ) , false ) ,
1145
+ Token :: make_word( "ea" , None ) ,
1146
+ Token :: Comma ,
1147
+ Token :: Whitespace ( Whitespace :: Space ) ,
1148
+ Token :: Number ( String :: from( "1e-10" ) , false ) ,
1149
+ Token :: make_word( "a" , None ) ,
1150
+ Token :: Comma ,
1151
+ Token :: Whitespace ( Whitespace :: Space ) ,
1152
+ Token :: Number ( String :: from( "1e-10" ) , false ) ,
1153
+ Token :: Minus ,
1154
+ Token :: Number ( String :: from( "10" ) , false ) ,
1155
+ ] ;
1156
+
1157
+ compare ( expected, tokens) ;
1158
+ }
1159
+
1094
1160
#[ test]
1095
1161
fn tokenize_scalar_function ( ) {
1096
1162
let sql = String :: from ( "SELECT sqrt(1)" ) ;
0 commit comments