BUG: Standardize malformed row handling in Python engine

Closes pandas-devgh-15910.

Author: gfyoung

pandas/io/parsers.py

@@ -2469,26 +2469,7 @@ def _next_line(self):
                 next(self.data)
 
             while True:
-                try:
-                    orig_line = next(self.data)
-                except csv.Error as e:
-                    msg = str(e)
-
-                    if 'NULL byte' in str(e):
-                        msg = ('NULL byte detected. This byte '
-                               'cannot be processed in Python\'s '
-                               'native csv library at the moment, '
-                               'so please pass in engine=\'c\' instead')
-
-                    if self.skipfooter > 0:
-                        reason = ('Error could possibly be due to '
-                                  'parsing errors in the skipped footer rows '
-                                  '(the skipfooter keyword is only applied '
-                                  'after Python\'s csv library has parsed '
-                                  'all rows).')
-                        msg += '. ' + reason
-
-                    raise csv.Error(msg)
+                orig_line = self._next_iter_line(self.data)
                 line = self._check_comments([orig_line])[0]
                 self.pos += 1
                 if (not self.skip_blank_lines and
@@ -2510,6 +2491,45 @@ def _next_line(self):
         self.buf.append(line)
         return line
 
+    def _next_iter_line(self, source, **kwargs):
+        """
+        Wrapper around iterating through a CSV source.
+
+        When a CSV error is raised, we check for specific
+        error messages that allow us to customize the
+        error message displayed to the user.
+
+        Parameters
+        ----------
+        source : The CSV source through which to iterate.
+        kwargs : Keyword arguments used to customize the error message.
+        """
+
+        try:
+            return next(source)
+        except csv.Error as e:
+            msg = str(e)
+
+            if 'NULL byte' in msg:
+                msg = ('NULL byte detected. This byte '
+                       'cannot be processed in Python\'s '
+                       'native csv library at the moment, '
+                       'so please pass in engine=\'c\' instead')
+            elif 'newline inside string' in msg:
+                msg = ('EOF inside string starting with '
+                       'line ' + str(kwargs['row_num']))
+                raise Exception(msg)
+
+            if self.skipfooter > 0:
+                reason = ('Error could possibly be due to '
+                          'parsing errors in the skipped footer rows '
+                          '(the skipfooter keyword is only applied '
+                          'after Python\'s csv library has parsed '
+                          'all rows).')
+                msg += '. ' + reason
+
+            raise csv.Error(msg)
+
     def _check_comments(self, lines):
         if self.comment is None:
             return lines
@@ -2730,17 +2750,13 @@ def _get_lines(self, rows=None):
                         lines.extend(new_rows)
                     else:
                         rows = 0
+
                         while True:
-                            try:
-                                new_rows.append(next(source))
-                                rows += 1
-                            except csv.Error as inst:
-                                if 'newline inside string' in str(inst):
-                                    row_num = str(self.pos + rows)
-                                    msg = ('EOF inside string starting with '
-                                           'line ' + row_num)
-                                    raise Exception(msg)
-                                raise
+                            new_row = self._next_iter_line(
+                                source, row_num=self.pos + rows)
+                            new_rows.append(new_row)
+                            rows += 1
+
                 except StopIteration:
                     if self.skiprows:
                         new_rows = [row for i, row in enumerate(new_rows)

pandas/tests/io/parser/c_parser_only.py

@@ -408,3 +408,12 @@ def test_large_difference_in_columns(self):
         expected = DataFrame([row.split(',')[0] for row in rows])
 
         tm.assert_frame_equal(result, expected)
+
+    def test_data_after_quote(self):
+        # see gh-15910
+
+        data = 'a\n1\n"b"a'
+        result = self.read_csv(StringIO(data))
+        expected = DataFrame({'a': ['1', 'ba']})
+
+        tm.assert_frame_equal(result, expected)

pandas/tests/io/parser/python_parser_only.py

@@ -225,15 +225,17 @@ def test_multi_char_sep_quotes(self):
 
     def test_skipfooter_bad_row(self):
         # see gh-13879
+        # see gh-15910
 
-        data = 'a,b,c\ncat,foo,bar\ndog,foo,"baz'
         msg = 'parsing errors in the skipped footer rows'
 
-        with tm.assertRaisesRegexp(csv.Error, msg):
-            self.read_csv(StringIO(data), skipfooter=1)
-
-        # We expect no match, so there should be an assertion
-        # error out of the inner context manager.
-        with tm.assertRaises(AssertionError):
+        for data in ('a\n1\n"b"a',
+                     'a,b,c\ncat,foo,bar\ndog,foo,"baz'):
             with tm.assertRaisesRegexp(csv.Error, msg):
-                self.read_csv(StringIO(data))
+                self.read_csv(StringIO(data), skipfooter=1)
+
+            # We expect no match, so there should be an assertion
+            # error out of the inner context manager.
+            with tm.assertRaises(AssertionError):
+                with tm.assertRaisesRegexp(csv.Error, msg):
+                    self.read_csv(StringIO(data))