ENH: preliminary Cython code and test type conversion

Author: wesm

pandas/io/tests/test_cparser.py

@@ -0,0 +1,66 @@
+"""
+C/Cython ascii file parser tests
+"""
+
+from pandas.util.py3compat import StringIO, BytesIO
+from datetime import datetime
+import csv
+import os
+import sys
+import re
+import unittest
+
+import nose
+
+from numpy import nan
+import numpy as np
+
+from pandas import DataFrame, Series, Index, isnull, MultiIndex
+import pandas.io.parsers as parsers
+from pandas.io.parsers import (read_csv, read_table, read_fwf,
+                               ExcelFile, TextParser)
+from pandas.util.testing import (assert_almost_equal, assert_frame_equal,
+                                 assert_series_equal, network)
+import pandas.lib as lib
+from pandas.util import py3compat
+from pandas.lib import Timestamp
+from pandas.tseries.index import date_range
+
+
+import pandas._parser as parser
+
+
+def curpath():
+    pth, _ = os.path.split(os.path.abspath(__file__))
+    return pth
+
+class TestCParser(unittest.TestCase):
+
+    def setUp(self):
+        self.dirpath = curpath()
+        self.csv1 = os.path.join(self.dirpath, 'test1.csv')
+        self.csv2 = os.path.join(self.dirpath, 'test2.csv')
+        self.xls1 = os.path.join(self.dirpath, 'test.xls')
+
+    def test_string_filename(self):
+        reader = parser.TextReader(self.csv1)
+        result = reader.read()
+
+    def test_file_handle(self):
+        try:
+            f = open(self.csv1, 'rb')
+            reader = parser.TextReader(f)
+            result = reader.read()
+        finally:
+            f.close()
+
+    # def test_StringIO(self):
+    #     text = open(self.csv1, 'rb').read()
+
+    #     reader = parser.TextReader(BytesIO(text))
+    #     result = reader.read()
+
+if __name__ == '__main__':
+    nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],
+                   exit=False)
+

pandas/src/parser.pyx

@@ -1,7 +1,19 @@
+cimport numpy as cnp
+import numpy as np
+
+cnp.import_array()
+
 cdef extern from "Python.h":
     ctypedef struct FILE
     FILE* PyFile_AsFile(object)
 
+cdef extern from "parser/conversions.h":
+    inline int to_double(char *item, double *p_value,
+                         char sci, char decimal)
+    inline int to_complex(char *item, double *p_real,
+                          double *p_imag, char sci, char decimal)
+    inline int to_longlong(char *item, long long *p_value)
+
 
 cdef extern from "parser/common.h":
 
@@ -91,6 +103,15 @@ cdef extern from "parser/common.h":
         #  error handling
         char *error_msg
 
+    ctypedef struct coliter_t:
+        char **words
+        int *line_start
+        int col
+        int line
+
+    void coliter_setup(coliter_t *it, parser_t *parser, int i)
+    char* COLITER_NEXT(coliter_t it)
+
     parser_t* parser_new()
 
     int parser_init(parser_t *self)
@@ -107,6 +128,8 @@ cdef extern from "parser/common.h":
     int tokenize_all_rows(parser_t *self)
     int tokenize_nrows(parser_t *self, size_t nrows)
 
+DEFAULT_CHUNKSIZE = 256 * 1024
+
 cdef class TextReader:
     '''
 
@@ -118,21 +141,29 @@ cdef class TextReader:
         parser_t *parser
         object file_handle, should_close
 
+    cdef public:
+        object delimiter, na_values, converters, thousands, delim_whitespace
+
     def __cinit__(self, source, delimiter=',', header=0, memory_map=False,
+                  chunksize=DEFAULT_CHUNKSIZE,
                   delim_whitespace=False,
                   na_values=None,
                   converters=None,
                   thousands=None):
         self.parser = parser_new()
+        self.parser.chunksize = chunksize
 
         self._setup_parser_source(source)
+        set_parser_default_options(self.parser)
+
+        parser_init(self.parser)
 
         if delim_whitespace:
             raise NotImplementedError
         else:
             if len(delimiter) > 1:
                 raise ValueError('only length-1 separators excluded right now')
-            self.parser.delimiter = delimiter
+            self.parser.delimiter = (<char*> delimiter)[0]
 
         # TODO: no header vs. header is not the first row
         self.parser.header = header
@@ -142,7 +173,7 @@ cdef class TextReader:
         self.delimiter = delimiter
         self.delim_whitespace = delim_whitespace
 
-        self.na_values
+        self.na_values = na_values
         self.converters = converters
         self.thousands = thousands
 
@@ -158,12 +189,13 @@ cdef class TextReader:
 
         if isinstance(source, (basestring, file)):
             if isinstance(source, basestring):
-                self.file_handle = open(source, 'rb')
+                source = open(source, 'rb')
                 self.should_close = True
-                source = self.file_handle
 
+            self.file_handle = source
             status = parser_file_source_init(self.parser,
                                              PyFile_AsFile(source))
+
             if status != 0:
                 raise Exception('Initializing from file failed')
 
@@ -189,20 +221,70 @@ cdef class TextReader:
         """
         rows=None --> read all rows
         """
-        cdef int status
+        cdef:
+            int prior_lines
+            int status
 
         if rows is not None:
             raise NotImplementedError
         else:
             status = tokenize_all_rows(self.parser)
 
+        if status < 0:
+            raise_parser_error('Error tokenizing data', self.parser)
+
+        result = self._convert_column_data()
+
+        # debug_print_parser(self.parser)
+        return result
+
+    def _convert_column_data(self):
+        cdef:
+            Py_ssize_t i, ncols
+
+        ncols = self.parser.line_fields[0]
+
+        results = {}
+        for i in range(ncols):
+            col_res = _try_double(self.parser, i, 0, self.parser.lines)
+
+            results[i] = col_res
+
+        return results
 
 class CParserError(Exception):
     pass
 
 
+cdef _try_double(parser_t *parser, int col, int line_start, int line_end):
+    cdef:
+        int error
+        size_t i, lines
+        coliter_t it
+        char *word
+        double *data
+        cnp.ndarray result
+
+    lines = line_end - line_start
+
+    result = np.empty(lines, dtype=np.float64)
+
+    data = <double *> result.data
+
+    coliter_setup(&it, parser, col)
+    for i in range(lines):
+        word = COLITER_NEXT(it)
+        error = to_double(word, data, parser.sci, parser.decimal)
+
+        if error != 1:
+            return None
+
+        data += 1
+
+    return result
+
 cdef raise_parser_error(object base, parser_t *parser):
-    message = '%s. C error: '
+    message = '%s. C error: ' % base
     if parser.error_msg != NULL:
         message += parser.error_msg
     else:

pandas/src/parser/common.h

@@ -178,7 +178,8 @@ typedef struct coliter_t {
     int line;
 } coliter_t;
 
-
+void coliter_setup(coliter_t *self, parser_t *parser, int i);
+coliter_t *coliter_new(parser_t *self, int i);
 
 /* #define COLITER_NEXT(iter) iter->words[iter->line_start[iter->line++] + iter->col] */
 #define COLITER_NEXT(iter) iter.words[iter.line_start[iter.line++] + iter.col]

pandas/src/parser/rows.c

@@ -130,6 +130,7 @@ void del_file_source(void *fs) {
 	// TODO: error codes?
 	// fclose(FS(fs)->fp);
 
+	// fseek(FS(fs)->fp, FS(fs)->initial_file_pos, SEEK_SET);
 	// allocated on the heap
 	free(fs);
 }
@@ -621,7 +622,7 @@ int convert_infer(parser_t *parser, array_t* result,
 
 void set_parser_default_options(parser_t *self) {
     // File buffer preferences
-    self->sourcetype = 'F';
+    // self->sourcetype = 'F';
 
     // parsing, type inference
     self->infer_types = 1;
@@ -659,10 +660,11 @@ parser_t* parser_new() {
 }
 
 int parser_file_source_init(parser_t *self, FILE* fp) {
-	FS(self->source)->fp = fp;
+	self->sourcetype = 'F';
+	self->source = new_file_source(fp);
 
 	// Only allocate this heap memory if we are not memory-mapping the file
-	self->data = (char*) malloc(self->chunksize * sizeof(char));
+	self->data = (char*) malloc((self->chunksize + 1) * sizeof(char));
 
 	if (self->data == NULL) {
 		return PARSER_OUT_OF_MEMORY;
@@ -680,7 +682,9 @@ int parser_gzip_source_init(parser_t *self, FILE* fp) {
 }
 
 int parser_array_source_init(parser_t *self, char *bytes, size_t length) {
+	self->sourcetype = 'A';
 	self->source = new_array_source(bytes, length);
+	return 0;
 }
 
 int parser_init(parser_t *self) {
@@ -959,15 +963,15 @@ int parser_cleanup(parser_t *self) {
 
 int parser_buffer_bytes(parser_t *self, size_t nbytes) {
     size_t bytes;
+	void *src = self->source;
 
 	// This should probably end up as a method table
 
 	switch(self->sourcetype) {
 		case 'F': // basic FILE*
 
-			bytes = fread((void *) self->data,
-						  sizeof(char), nbytes,
-						  FS(self->source)->fp);
+			bytes = fread((void *) self->data, sizeof(char), nbytes,
+						  FS(src)->fp);
 			self->datalen = bytes;
 
 			TRACE(("Read %d bytes\n", (int) bytes));
@@ -980,6 +984,24 @@ int parser_buffer_bytes(parser_t *self, size_t nbytes) {
 			break;
 
 		case 'A': // in-memory bytes (e.g. from StringIO)
+			if (ARS(src)->position == ARS(src)->length) {
+				return REACHED_EOF;
+			}
+
+			self->data = ARS(src)->data + ARS(src)->position;
+
+			if (ARS(src)->position + nbytes > ARS(src)->length) {
+				// fewer than nbytes remaining
+				self->datalen = ARS(src)->length - ARS(src)->position;
+			} else {
+				self->datalen = nbytes;
+			}
+
+			ARS(src)->position += self->datalen;
+
+			TRACE(("datalen: %d\n", self->datalen));
+
+			TRACE(("pos: %d, length: %d", ARS(src)->position, ARS(src)->length));
 
 			break;
 
@@ -1310,6 +1332,8 @@ int _tokenize_helper(parser_t *self, size_t nrows, int all) {
 
         status = parser_buffer_bytes(self, self->chunksize);
 
+		TRACE(("sourcetype: %c, status: %d\n", self->sourcetype, status));
+
         if (status == REACHED_EOF) {
 			// XXX close last line
 			status = parser_handle_eof(self);