ENH: add comments keyword to text readers. #962

Author: Chang She

pandas/io/parsers.py

@@ -55,6 +55,8 @@ def next(x):
     DD/MM format dates, international and European format
 thousands : str, default None
     Thousands separator
+comment : str, default None
+    Indicates remainder of line should not be parsed
 nrows : int, default None
     Number of rows of file to read. Useful for reading pieces of large files
 iterator : boolean, default False
@@ -179,6 +181,7 @@ def read_csv(filepath_or_buffer,
              skiprows=None,
              na_values=None,
              thousands=None,
+             comment=None,
              parse_dates=False,
              dayfirst=False,
              date_parser=None,
@@ -208,6 +211,7 @@ def read_table(filepath_or_buffer,
                skiprows=None,
                na_values=None,
                thousands=None,
+               comment=None,
                parse_dates=False,
                dayfirst=False,
                date_parser=None,
@@ -241,6 +245,7 @@ def read_fwf(filepath_or_buffer,
              skiprows=None,
              na_values=None,
              thousands=None,
+             comment=None,
              parse_dates=False,
              dayfirst=False,
              date_parser=None,
@@ -339,6 +344,10 @@ class TextParser(object):
         Column or columns to use as the (possibly hierarchical) index
     na_values : iterable, default None
         Custom NA values
+    thousands : str, default None
+        Thousands separator
+    comment : str, default None
+        Comment out remainder of line
     parse_dates : boolean, default False
     date_parser : function, default None
     skiprows : list of integers
@@ -351,7 +360,7 @@ class TextParser(object):
 
     def __init__(self, f, delimiter=None, names=None, header=0,
                  index_col=None, na_values=None, thousands=None,
-                 parse_dates=False,
+                 comment=None, parse_dates=False,
                  date_parser=None, dayfirst=False, chunksize=None,
                  skiprows=None, skip_footer=0, converters=None,
                  verbose=False, encoding=None):
@@ -398,6 +407,7 @@ def __init__(self, f, delimiter=None, names=None, header=0,
             self.na_values = set(list(na_values)) | _NA_VALUES
 
         self.thousands = thousands
+        self.comment = comment
 
         if hasattr(f, 'readline'):
             self._make_reader(f)
@@ -430,6 +440,12 @@ def _make_reader(self, f):
                     self.pos += 1
                     line = f.readline()
 
+                while self._is_commented(line):
+                    self.pos += 1
+                    line = f.readline()
+
+                line = self._check_comments([line])[0]
+
                 self.pos += 1
                 sniffed = csv.Sniffer().sniff(line)
                 dia.delimiter = sniffed.delimiter
@@ -498,22 +514,56 @@ def _next_line(self):
                 self.pos += 1
 
             try:
-                line = self.data[self.pos]
+                while True:
+                    line = self.data[self.pos]
+                    if not self._is_commented(line):
+                        break
+                    self.pos += 1
             except IndexError:
                 raise StopIteration
         else:
             while self.pos in self.skiprows:
                 next(self.data)
                 self.pos += 1
-            line = next(self.data)
 
+            while True:
+                line = next(self.data)
+                if not self._is_commented(line):
+                    break
+                self.pos += 1
+
+        line = self._check_comments([line])[0]
         line = self._check_thousands([line])[0]
 
         self.pos += 1
         self.buf.append(line)
 
         return line
 
+    def _is_commented(self, line):
+        if self.comment is None or len(line) == 0:
+            return False
+        return line[0].startswith(self.comment)
+
+    def _check_comments(self, lines):
+        if self.comment is None:
+            return lines
+        ret = []
+        for l in lines:
+            rl = []
+            for x in l:
+                if (not isinstance(x, basestring) or
+                    self.comment not in x):
+                    rl.append(x)
+                else:
+                    x = x[:x.find(self.comment)]
+                    if len(x) > 0:
+                        rl.append(x)
+                    break
+            if len(rl) > 0:
+                ret.append(rl)
+        return ret
+
     def _check_thousands(self, lines):
         if self.thousands is None:
             return lines
@@ -730,6 +780,7 @@ def _get_lines(self, rows=None):
         if self.skip_footer:
             lines = lines[:-self.skip_footer]
 
+        lines = self._check_comments(lines)
         return self._check_thousands(lines)
 
 def _convert_to_ndarrays(dct, na_values, verbose=False):

pandas/io/tests/test_parsers.py

@@ -66,6 +66,32 @@ def test_1000_fwf(self):
                       thousands=',')
         assert_almost_equal(df.values, expected)
 
+    def test_comment(self):
+        data = """A,B,C
+1,2.,4.#hello world
+#hello self
+5.,NaN,10.0
+"""
+        expected = [[1., 2., 4.],
+                    [5., np.nan, 10.]]
+        df = read_csv(StringIO(data), comment='#')
+        assert_almost_equal(df.values, expected)
+
+        df = read_table(StringIO(data), sep=',', comment='#', na_values=['NaN'])
+        assert_almost_equal(df.values, expected)
+
+    def test_comment_fwf(self):
+        data = """
+  1   2.   4  #hello world
+#hello self
+  5  NaN  10.0
+"""
+        expected = [[1, 2., 4],
+                    [5, np.nan, 10.]]
+        df = read_fwf(StringIO(data), colspecs=[(0,3),(4,9),(9,25)],
+                      comment='#')
+        assert_almost_equal(df.values, expected)
+
     def test_custom_na_values(self):
         data = """A,B,C
 ignore,this,row

vb_suite/parser.py

@@ -10,10 +10,10 @@
 N = 10000
 K = 8
 df = DataFrame(np.random.randn(N, K) * np.random.randint(100, 10000, (N, K)))
-df.to_csv('test.csv')
+df.to_csv('test.csv', sep='|')
 """
 
-read_csv_vb = Benchmark("read_csv('test.csv')", setup,
+read_csv_vb = Benchmark("read_csv('test.csv', sep='|')", setup,
                         cleanup="os.remove('test.csv')",
                         start_date=datetime(2012, 5, 7))
 
@@ -29,6 +29,24 @@
 df.to_csv('test.csv', sep='|')
 """
 
-read_csv_thou_vb = Benchmark("read_csv('test.csv')", setup,
+read_csv_thou_vb = Benchmark("read_csv('test.csv', sep='|', thousands=',')",
+                             setup,
                              cleanup="os.remove('test.csv')",
                              start_date=datetime(2012, 5, 7))
+
+setup = common_setup + """
+from pandas import read_csv
+import os
+N = 10000
+K = 8
+format = lambda x: '%f' % x
+df = DataFrame(np.random.randn(N, K) * np.random.randint(100, 10000, (N, K)))
+df = df.applymap(format)
+df.ix[:5, 0] = '#'
+df.to_csv('test.csv', sep='|')
+"""
+
+read_csv_comment_vb = Benchmark("read_csv('test.csv', sep='|', comment='#')",
+                                setup,
+                                cleanup="os.remove('test.csv')",
+                                start_date=datetime(2012, 5, 7))