Added casting

Author: TomAugspurger

pandas/_libs/parsers.pyx

@@ -48,7 +48,7 @@ from pandas.core.dtypes.common import (
 from pandas.core.categorical import Categorical
 from pandas.core.algorithms import take_1d
 from pandas.core.dtypes.concat import union_categoricals
-from pandas import Index
+from pandas import Index, to_numeric, to_datetime, to_timedelta
 
 import pandas.io.common as com
 
@@ -1267,12 +1267,30 @@ cdef class TextReader:
             return self._string_convert(i, start, end, na_filter,
                                         na_hashset)
         elif is_categorical_dtype(dtype):
-            # TODO: I suspect that this could be optimized when dtype
-            # is an instance of CategoricalDtype
+            # TODO: I suspect that _categorical_convert could be
+            # optimized when dtype is an instance of CategoricalDtype
             codes, cats, na_count = _categorical_convert(
                 self.parser, i, start, end, na_filter,
                 na_hashset, self.c_encoding)
             cats = Index(cats)
+
+            # Here is where we'll do the casting...
+            if (isinstance(dtype, CategoricalDtype) and
+                    dtype.categories is not None):
+                if dtype.categories.is_numeric():
+                    # is ignore correct?
+                    cats = to_numeric(cats, errors='ignore')
+                elif dtype.categories.is_all_dates:
+                    # is ignore correct?
+                    if is_datetime64_dtype(dtype.categories):
+                        print("before", cats)
+                        cats = to_datetime(cats, errors='ignore')
+                        print("after", cats)
+                    else:
+                        print("before", cats)
+                        cats = to_timedelta(cats, errors='ignore')
+                        print("after", cats)
+
             if (isinstance(dtype, CategoricalDtype) and
                     dtype.categories is not None):
                 # recode for dtype.categories

pandas/io/parsers.py

@@ -12,15 +12,16 @@
 
 import numpy as np
 
-from pandas import compat
+from pandas import compat, to_numeric, to_timedelta
 from pandas.compat import (range, lrange, PY3, StringIO, lzip,
                            zip, string_types, map, u)
 from pandas.core.dtypes.common import (
     is_integer, _ensure_object,
     is_list_like, is_integer_dtype,
     is_float, is_dtype_equal,
     is_object_dtype, is_string_dtype,
-    is_scalar, is_categorical_dtype)
+    is_scalar, is_categorical_dtype,
+    is_datetime64_dtype, is_timedelta64_dtype)
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from pandas.core.dtypes.missing import isna
 from pandas.core.dtypes.cast import astype_nansafe
@@ -1606,11 +1607,21 @@ def _cast_types(self, values, cast_type, column):
             # XXX this is for consistency with
             # c-parser which parses all categories
             # as strings
-            if not is_object_dtype(values):
-                values = astype_nansafe(values, str)
-            if isinstance(cast_type, CategoricalDtype):
+            known_cats = (isinstance(cast_type, CategoricalDtype) and
+                          cast_type.categories is not None)
+            str_values = is_object_dtype(values)
+
+            if known_cats and str_values:
+                if cast_type.categories.is_numeric():
+                    values = to_numeric(values, errors='ignore')
+                elif is_datetime64_dtype(cast_type.categories):
+                    values = tools.to_datetime(values, errors='ignore')
+                elif is_timedelta64_dtype(cast_type.categories):
+                    values = to_timedelta(values, errors='ignore')
                 values = Categorical(values, categories=cast_type.categories,
                                      ordered=cast_type.ordered)
+            elif not is_object_dtype(values):
+                values = astype_nansafe(values, str)
             else:
                 values = Categorical(values)
         else:

pandas/tests/io/parser/dtypes.py

@@ -187,20 +187,28 @@ def test_categorical_categoricaldtype_unsorted(self):
         result = self.read_csv(StringIO(data), dtype={'b': dtype})
         tm.assert_frame_equal(result, expected)
 
-#     @pytest.mark.parametrize('ordered', [True, False])
-#     def test_categoricaldtype_coerces(self, ordered):
-#         dtype = {'b': CategoricalDtype([10, 11, 12, 13], ordered=ordered)}
-#         data = """a,b
-# 1,10
-# 1,11
-# 1,12
-# 2,13"""
-#         expected = pd.DataFrame({
-#             'a': [1, 1, 1, 2],
-#             'b': Categorical([10, 11, 12, 13], ordered=ordered),
-#         }, columns=['a', 'b'])
-#         result = self.read_csv(StringIO(data), dtype=dtype)
-#         tm.assert_frame_equal(result, expected)
+    def test_categoricaldtype_coerces_numeric(self):
+        dtype = {'b': CategoricalDtype([1, 2, 3])}
+        data = "b\n1\n1\n2\n3"
+        expected = pd.DataFrame({'b': Categorical([1, 1, 2, 3])})
+        result = self.read_csv(StringIO(data), dtype=dtype)
+        tm.assert_frame_equal(result, expected)
+
+    def test_categoricaldtype_coerces_datetime(self):
+        dtype = {
+            'b': CategoricalDtype(pd.date_range('2017', '2019', freq='AS'))
+        }
+        data = "b\n2017-01-01\n2018-01-01\n2019-01-01"
+        expected = pd.DataFrame({'b': Categorical(dtype['b'].categories)})
+        result = self.read_csv(StringIO(data), dtype=dtype)
+        tm.assert_frame_equal(result, expected)
+
+    def test_categoricaldtype_coerces_timedelta(self):
+        dtype = {'b': CategoricalDtype(pd.to_timedelta(['1H', '2H', '3H']))}
+        data = "b\n1H\n2H\n3H"
+        expected = pd.DataFrame({'b': Categorical(dtype['b'].categories)})
+        result = self.read_csv(StringIO(data), dtype=dtype)
+        tm.assert_frame_equal(result, expected)
 
     def test_categorical_categoricaldtype_chunksize(self):
         # GH 10153