Report an error for unterminated string literals

...updated the TODOs regarding single-quoted literals parsing while at it.

Author: nickolay

src/parser.rs

@@ -1324,6 +1324,16 @@ impl Parser {
             }
             // MSSQL supports single-quoted strings as aliases for columns
             // We accept them as table aliases too, although MSSQL does not.
+            //
+            // Note, that this conflicts with an obscure rule from the SQL
+            // standard, which we don't implement:
+            // https://crate.io/docs/sql-99/en/latest/chapters/07.html#character-string-literal-s
+            //    "[Obscure Rule] SQL allows you to break a long <character
+            //    string literal> up into two or more smaller <character string
+            //    literal>s, split by a <separator> that includes a newline
+            //    character. When it sees such a <literal>, your DBMS will
+            //    ignore the <separator> and treat the multiple strings as
+            //    a single <literal>."
             Some(Token::SingleQuotedString(ref s)) => Ok(Some(Ident::with_quote('\'', s.clone()))),
             not_an_ident => {
                 if after_as {

src/tokenizer.rs

@@ -278,7 +278,7 @@ impl<'a> Tokenizer<'a> {
                     match chars.peek() {
                         Some('\'') => {
                             // N'...' - a <national character string literal>
-                            let s = self.tokenize_single_quoted_string(chars);
+                            let s = self.tokenize_single_quoted_string(chars)?;
                             Ok(Some(Token::NationalStringLiteral(s)))
                         }
                         _ => {
@@ -295,7 +295,7 @@ impl<'a> Tokenizer<'a> {
                     match chars.peek() {
                         Some('\'') => {
                             // X'...' - a <binary string literal>
-                            let s = self.tokenize_single_quoted_string(chars);
+                            let s = self.tokenize_single_quoted_string(chars)?;
                             Ok(Some(Token::HexStringLiteral(s)))
                         }
                         _ => {
@@ -313,7 +313,7 @@ impl<'a> Tokenizer<'a> {
                 }
                 // string
                 '\'' => {
-                    let s = self.tokenize_single_quoted_string(chars);
+                    let s = self.tokenize_single_quoted_string(chars)?;
                     Ok(Some(Token::SingleQuotedString(s)))
                 }
                 // delimited (quoted) identifier
@@ -431,11 +431,10 @@ impl<'a> Tokenizer<'a> {
     }
 
     /// Read a single quoted string, starting with the opening quote.
-    fn tokenize_single_quoted_string(&self, chars: &mut Peekable<Chars<'_>>) -> String {
-        //TODO: handle escaped quotes in string
-        //TODO: handle newlines in string
-        //TODO: handle EOF before terminating quote
-        //TODO: handle 'string' <white space> 'string continuation'
+    fn tokenize_single_quoted_string(
+        &self,
+        chars: &mut Peekable<Chars<'_>>,
+    ) -> Result<String, TokenizerError> {
         let mut s = String::new();
         chars.next(); // consume the opening quote
         while let Some(&ch) = chars.peek() {
@@ -447,7 +446,7 @@ impl<'a> Tokenizer<'a> {
                         s.push('\'');
                         chars.next();
                     } else {
-                        break;
+                        return Ok(s);
                     }
                 }
                 _ => {
@@ -456,7 +455,10 @@ impl<'a> Tokenizer<'a> {
                 }
             }
         }
-        s
+        Err(TokenizerError(format!(
+            "Unterminated string literal at Line: {}, Col: {}",
+            self.line, self.col
+        )))
     }
 
     fn tokenize_multiline_comment(
@@ -640,6 +642,31 @@ mod tests {
         compare(expected, tokens);
     }
 
+    #[test]
+    fn tokenize_newline_in_string_literal() {
+        let sql = String::from("'foo\r\nbar\nbaz'");
+
+        let dialect = GenericDialect {};
+        let mut tokenizer = Tokenizer::new(&dialect, &sql);
+        let tokens = tokenizer.tokenize().unwrap();
+        let expected = vec![Token::SingleQuotedString("foo\r\nbar\nbaz".to_string())];
+        compare(expected, tokens);
+    }
+
+    #[test]
+    fn tokenize_unterminated_string_literal() {
+        let sql = String::from("select 'foo");
+
+        let dialect = GenericDialect {};
+        let mut tokenizer = Tokenizer::new(&dialect, &sql);
+        assert_eq!(
+            tokenizer.tokenize(),
+            Err(TokenizerError(
+                "Unterminated string literal at Line: 1, Col: 8".to_string()
+            ))
+        );
+    }
+
     #[test]
     fn tokenize_invalid_string_cols() {
         let sql = String::from("\n\nSELECT * FROM table\tمصطفىh");

tests/sqlparser_common.rs

@@ -1031,7 +1031,7 @@ fn parse_create_external_table() {
                name VARCHAR(100) NOT NULL,\
                lat DOUBLE NULL,\
                lng DOUBLE)\
-               STORED AS TEXTFILE LOCATION '/tmp/example.csv";
+               STORED AS TEXTFILE LOCATION '/tmp/example.csv'";
     let ast = one_statement_parses_to(
         sql,
         "CREATE EXTERNAL TABLE uk_cities (\