ENH: missing trailing fields become NA, and tested to verify that's the case. close #2430

Author: wesm

pandas/io/tests/test_cparser.py

@@ -137,12 +137,13 @@ def test_integer_thousands(self):
         tm.assert_almost_equal(result[0], expected)
 
     def test_skip_bad_lines(self):
+        # too many lines, see #2430 for why
         data = ('a:b:c\n'
                 'd:e:f\n'
                 'g:h:i\n'
-                'j:k\n'
+                'j:k:l:m\n'
                 'l:m:n\n'
-                'o:p')
+                'o:p:q:r')
 
         reader = TextReader(StringIO(data), delimiter=':',
                             header=None)

pandas/io/tests/test_parsers.py

@@ -720,10 +720,11 @@ def test_read_table_unicode(self):
         self.assert_(isinstance(df1['X0'].values[0], unicode))
 
     def test_read_table_wrong_num_columns(self):
+        # too few!
         data = """A,B,C,D,E,F
-1,2,3,4,5
-6,7,8,9,10
-11,12,13,14,15
+1,2,3,4,5,6
+6,7,8,9,10,11,12
+11,12,13,14,15,16
 """
         self.assertRaises(Exception, self.read_csv, StringIO(data))
 
@@ -1302,6 +1303,15 @@ def test_nonexistent_path(self):
         path = '%s.csv' % tm.rands(10)
         self.assertRaises(Exception, self.read_csv, path)
 
+    def test_missing_trailing_delimiters(self):
+        data = """A,B,C,D
+1,2,3,4
+1,3,3,
+1,4,5"""
+        result = self.read_csv(StringIO(data))
+        self.assertTrue(result['D'].isnull()[1:].all())
+
+
 class TestPythonParser(ParserTests, unittest.TestCase):
 
     def read_csv(self, *args, **kwds):
@@ -1645,6 +1655,7 @@ def test_utf16_example(self):
             result = self.read_table(buf, encoding='utf-16')
             self.assertEquals(len(result), 50)
 
+
 class TestCParserHighMemory(ParserTests, unittest.TestCase):
 
     def read_csv(self, *args, **kwds):

pandas/src/parser/tokenizer.c

@@ -447,7 +447,7 @@ static int end_line(parser_t *self) {
         }
     }
 
-    if (!(self->lines <= self->header + 1) && fields != ex_fields) {
+    if (!(self->lines <= self->header + 1) && fields > ex_fields) {
         // increment file line count
         self->file_lines++;
 
@@ -478,7 +478,16 @@ static int end_line(parser_t *self) {
                 free(msg);
             }
         }
-    } else {
+    }
+    else {
+        /* missing trailing delimiters */
+        if (self->lines >= self->header + 1 && self->lines > 0) {
+            while (fields < ex_fields){
+                end_field(self);
+                fields++;
+            }
+        }
+
         // increment both line counts
         self->file_lines++;