ENH: handle comments in C tokenizer. add synthetic vbenchmark. close #1204

Author: wesm

pandas/io/parsers.py

@@ -244,7 +244,7 @@ def _read(filepath_or_buffer, kwds):
     'widths': None
 }
 
-_c_unsupported = set(['comment', 'skip_footer'])
+_c_unsupported = set(['skip_footer'])
 _python_unsupported = set(_c_parser_defaults.keys())
 
 
@@ -501,7 +501,7 @@ def _clean_options(self, options, engine):
 
         # C engine not supported yet
         if engine == 'c':
-            if (options['comment'] or options['skip_footer'] > 0):
+            if options['skip_footer'] > 0:
                 engine = 'python'
 
         if engine == 'c':

pandas/src/parser.pyx

@@ -73,6 +73,7 @@ cdef extern from "parser/tokenizer.h":
         QUOTE_IN_QUOTED_FIELD
         EAT_CRNL
         EAT_WHITESPACE
+        EAT_COMMENT
         FINISHED
 
     ctypedef void* (*io_callback)(void *src, size_t nbytes, size_t *bytes_read,
@@ -332,6 +333,11 @@ cdef class TextReader:
         self.parser.quotechar = ord(quotechar)
         self.parser.quoting = quoting
 
+        if comment is not None:
+            if len(comment) > 1:
+                raise ValueError('Only length-1 comment characters supported')
+            self.parser.commentchar = ord(comment)
+
         # error handling of bad lines
         self.parser.error_bad_lines = int(error_bad_lines)
         self.parser.warn_bad_lines = int(warn_bad_lines)

pandas/src/parser/tokenizer.c

@@ -255,6 +255,8 @@ int parser_init(parser_t *self) {
     self->error_msg = NULL;
     self->warn_msg = NULL;
 
+    self->commentchar = '\0';
+
     return 0;
 }
 
@@ -688,6 +690,10 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
                 /* save empty field */
                 END_FIELD();
             }
+            else if (c == self->commentchar) {
+                END_FIELD();
+                self->state = EAT_COMMENT;
+            }
             else {
                 /* begin new unquoted field */
                 if (self->quoting == QUOTE_NONNUMERIC)
@@ -726,6 +732,10 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
                 END_FIELD();
                 self->state = START_FIELD;
             }
+            else if (c == self->commentchar) {
+                END_FIELD();
+                self->state = EAT_COMMENT;
+            }
             else {
                 /* normal character - save in field */
                 PUSH_CHAR(c);
@@ -811,6 +821,14 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
             }
             break;
 
+        case EAT_COMMENT:
+            if (c == '\n') {
+                END_LINE();
+            } else if (c == '\r') {
+                self->state = EAT_CRNL;
+            }
+            break;
+
         default:
             break;
 
@@ -919,6 +937,10 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
             else if (IS_WHITESPACE(c)) {
                 self->state = EAT_WHITESPACE;
             }
+            else if (c == self->commentchar) {
+                END_FIELD();
+                self->state = EAT_COMMENT;
+            }
             else {
                 /* begin new unquoted field */
                 if (self->quoting == QUOTE_NONNUMERIC)
@@ -957,6 +979,10 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
                 END_FIELD();
                 self->state = EAT_WHITESPACE;
             }
+            else if (c == self->commentchar) {
+                END_FIELD();
+                self->state = EAT_COMMENT;
+            }
             else {
                 /* normal character - save in field */
                 PUSH_CHAR(c);
@@ -1042,6 +1068,14 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
             }
             break;
 
+        case EAT_COMMENT:
+            if (c == '\n') {
+                END_LINE();
+            } else if (c == '\r') {
+                self->state = EAT_CRNL;
+            }
+            break;
+
         default:
             break;
 

pandas/src/parser/tokenizer.h

@@ -119,6 +119,7 @@ typedef enum {
     QUOTE_IN_QUOTED_FIELD,
     EAT_CRNL,
     EAT_WHITESPACE,
+    EAT_COMMENT,
     FINISHED
 } ParserState;
 

vb_suite/parser.py

@@ -2,10 +2,10 @@
 from datetime import datetime
 
 common_setup = """from pandas_vb_common import *
+from pandas import read_csv, read_table
 """
 
 setup = common_setup + """
-from pandas import read_csv
 import os
 N = 10000
 K = 8
@@ -19,7 +19,6 @@
 
 
 setup = common_setup + """
-from pandas import read_csv
 import os
 N = 10000
 K = 8
@@ -35,7 +34,6 @@
                              start_date=datetime(2012, 5, 7))
 
 setup = common_setup + """
-from pandas import read_csv
 import os
 N = 10000
 K = 8
@@ -46,13 +44,22 @@
 df.to_csv('test.csv', sep='|')
 """
 
-read_csv_comment_vb = Benchmark("read_csv('test.csv', sep='|', comment='#')",
-                                setup,
-                                cleanup="os.remove('test.csv')",
-                                start_date=datetime(2012, 5, 7))
+read_csv_comment = Benchmark("read_csv('test.csv', sep='|', comment='#')",
+                             setup,
+                             cleanup="os.remove('test.csv')",
+                             start_date=datetime(2012, 5, 7))
+
+setup = common_setup + """
+data = ['A,B,C']
+data = data + ['1,2,3 # comment'] * 100000
+data = '\\n'.join(data)
+"""
+
+stmt = "read_csv(StringIO(data), comment='#')"
+read_csv_comment2 = Benchmark(stmt, setup,
+                              start_date=datetime(2011, 11, 1))
 
 setup = common_setup + """
-from pandas import read_table
 from cStringIO import StringIO
 import os
 N = 10000
@@ -72,7 +79,6 @@
 read_table_multiple_date = Benchmark(cmd, setup, start_date=sdate)
 
 setup = common_setup + """
-from pandas import read_table
 from cStringIO import StringIO
 import os
 N = 10000