BUG: use map_infer instead of np.vectorize. handle NA sentinels if converter yields numeric array, GH #753

Author: wesm

pandas/io/parsers.py

@@ -475,7 +475,7 @@ def get_chunk(self, rows=None):
         for col, f in self.converters.iteritems():
             if isinstance(col, int) and col not in self.columns:
                 col = self.columns[col]
-            result = np.vectorize(f)(data[col])
+            result = lib.map_infer(data[col], f)
             if issubclass(result.dtype.type, (basestring, unicode)):
                 result = result.astype('O')
             data[col] = result
@@ -533,6 +533,12 @@ def _convert_to_ndarrays(dct, na_values, verbose=False):
 def _convert_types(values, na_values):
     na_count = 0
     if issubclass(values.dtype.type, (np.number, np.bool_)):
+        mask = lib.ismember(values, na_values)
+        na_count = mask.sum()
+        if na_count > 0:
+            if com.is_integer_dtype(values):
+                values = values.astype(np.float64)
+            np.putmask(values, mask, np.nan)
         return values, na_count
 
     try:

pandas/io/tests/test_parsers.py

@@ -11,7 +11,7 @@
 from numpy import nan
 import numpy as np
 
-from pandas import DataFrame, Index
+from pandas import DataFrame, Index, isnull
 from pandas.io.parsers import read_csv, read_table, ExcelFile, TextParser
 from pandas.util.testing import assert_almost_equal, assert_frame_equal
 import pandas._tseries as lib
@@ -584,6 +584,61 @@ def test_read_csv_parse_simple_list(self):
                                        'foo', 'bar']})
         assert_frame_equal(df, expected)
 
+    def test_converters_corner_with_nas(self):
+        import StringIO
+        import numpy as np
+        import pandas
+        csv = """id,score,days
+1,2,12
+2,2-5,
+3,,14+
+4,6-12,2"""
+
+        def convert_days(x):
+           x = x.strip()
+           if not x: return np.nan
+
+           is_plus = x.endswith('+')
+           if is_plus:
+               x = int(x[:-1]) + 1
+           else:
+               x = int(x)
+           return x
+
+        def convert_days_sentinel(x):
+           x = x.strip()
+           if not x: return -1
+
+           is_plus = x.endswith('+')
+           if is_plus:
+               x = int(x[:-1]) + 1
+           else:
+               x = int(x)
+           return x
+
+        def convert_score(x):
+           x = x.strip()
+           if not x: return np.nan
+           if x.find('-')>0:
+               valmin, valmax = map(int, x.split('-'))
+               val = 0.5*(valmin + valmax)
+           else:
+               val = float(x)
+
+           return val
+
+        fh = StringIO.StringIO(csv)
+        result = pandas.read_csv(fh, converters={'score':convert_score,
+                                                 'days':convert_days},
+                                 na_values=[-1,'',None])
+        self.assert_(isnull(result['days'][1]))
+
+        fh = StringIO.StringIO(csv)
+        result2 = pandas.read_csv(fh, converters={'score':convert_score,
+                                                  'days':convert_days_sentinel},
+                                  na_values=[-1,'',None])
+        assert_frame_equal(result, result2)
+
 class TestParseSQL(unittest.TestCase):
 
     def test_convert_sql_column_floats(self):