BUG: fix tokenizer bug with \r line terminator and quoted fields. closes #3453

Author: wesm

RELEASE.rst

@@ -196,6 +196,7 @@ pandas 0.11.1
   - ``DataFrame.to_csv`` will succeed with the deprecated option ``nanRep``, @tdsmith
   - ``DataFrame.to_html`` and ``DataFrame.to_latex`` now accept a path for
     their first argument (GH3702_)
+  - Fix file tokenization error with \r delimiter and quoted fields (GH3453_)
 
 .. _GH3164: https://github.com/pydata/pandas/issues/3164
 .. _GH2786: https://github.com/pydata/pandas/issues/2786
@@ -220,6 +221,7 @@ pandas 0.11.1
 .. _GH3553: https://github.com/pydata/pandas/issues/3553
 .. _GH3437: https://github.com/pydata/pandas/issues/3437
 .. _GH3468: https://github.com/pydata/pandas/issues/3468
+.. _GH3453: https://github.com/pydata/pandas/issues/3453
 .. _GH3455: https://github.com/pydata/pandas/issues/3455
 .. _GH3457: https://github.com/pydata/pandas/issues/3457
 .. _GH3477: https://github.com/pydata/pandas/issues/3457

pandas/io/tests/test_parsers.py

@@ -2238,6 +2238,20 @@ def test_parse_ragged_csv(self):
 
         tm.assert_frame_equal(result, expected)
 
+    def test_tokenize_CR_with_quoting(self):
+        # #3453, this doesn't work with Python parser for some reason
+
+        data = ' a,b,c\r"a,b","e,d","f,f"'
+
+        result = self.read_csv(StringIO(data), header=None)
+        expected = self.read_csv(StringIO(data.replace('\r', '\n')),
+                                 header=None)
+        tm.assert_frame_equal(result, expected)
+
+        result = self.read_csv(StringIO(data))
+        expected = self.read_csv(StringIO(data.replace('\r', '\n')))
+        tm.assert_frame_equal(result, expected)
+
 
 class TestParseSQL(unittest.TestCase):
 

pandas/src/parser/tokenizer.c

@@ -687,6 +687,7 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
                self->state));
 
         switch(self->state) {
+
         case START_RECORD:
             // start of record
 
@@ -702,6 +703,7 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
             /* normal character - handle as START_FIELD */
             self->state = START_FIELD;
             /* fallthru */
+
         case START_FIELD:
             /* expecting field */
             if (c == '\n') {
@@ -846,6 +848,14 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
             }
             break;
 
+        case EAT_COMMENT:
+            if (c == '\n') {
+                END_LINE();
+            } else if (c == '\r') {
+                self->state = EAT_CRNL;
+            }
+            break;
+
         case EAT_CRNL:
             if (c == '\n') {
                 END_LINE();
@@ -854,16 +864,23 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
                 // Handle \r-delimited files
                 END_LINE_AND_FIELD_STATE(START_FIELD);
             } else {
-                PUSH_CHAR(c);
-                END_LINE_STATE(IN_FIELD);
-            }
-            break;
+                /* \r line terminator */
+
+                /* UGH. we don't actually want to consume the token. fix this later */
+                self->stream_len = slen;
+                if (end_line(self) < 0) {
+                    goto parsingerror;
+                }
+                stream = self->stream + self->stream_len;
+                slen = self->stream_len;
+                self->state = START_RECORD;
+
+                /* HACK, let's try this one again */
+                --i; buf--;
+                if (line_limit > 0 && self->lines == start_lines + line_limit) {
+                    goto linelimit;
+                }
 
-        case EAT_COMMENT:
-            if (c == '\n') {
-                END_LINE();
-            } else if (c == '\r') {
-                self->state = EAT_CRNL;
             }
             break;
 

pandas/src/parser/tokenizer.h

@@ -88,7 +88,7 @@ See LICENSE for the license
 #define ERROR_NO_DATA                  23
 
 
-// #define VERBOSE
+/* #define VERBOSE */
 
 #if defined(VERBOSE)
 #define TRACE(X) printf X;