COMPAT: invalid casting to nan

closes #12303

Author: jreback

pandas/core/common.py

@@ -1098,6 +1098,33 @@ def _infer_dtype_from_scalar(val):
     return dtype, val
 
 
+def _is_na_compat(arr, fill_value=np.nan):
+    """
+    Parameters
+    ----------
+    arr: a numpy array
+    fill_value: fill value, default to np.nan
+
+    Returns
+    -------
+    True if we can fill using this fill_value
+    """
+    dtype = arr.dtype
+    if isnull(fill_value):
+        return not (is_bool_dtype(dtype) or
+                    is_integer_dtype(dtype))
+    return True
+
+
+def _maybe_fill(arr, fill_value=np.nan):
+    """
+    if we have a compatiable fill_value and arr dtype, then fill
+    """
+    if _is_na_compat(arr, fill_value):
+        arr.fill(fill_value)
+    return arr
+
+
 def _maybe_promote(dtype, fill_value=np.nan):
 
     # if we passed an array here, determine the fill value by dtype
@@ -1359,7 +1386,10 @@ def trans(x):  # noqa
             # do a test on the first element, if it fails then we are done
             r = result.ravel()
             arr = np.array([r[0]])
-            if not np.allclose(arr, trans(arr).astype(dtype)):
+
+            # if we have any nulls, then we are done
+            if isnull(arr).any() or not np.allclose(arr,
+                                                    trans(arr).astype(dtype)):
                 return result
 
             # a comparable, e.g. a Decimal may slip in here

pandas/core/groupby.py

@@ -31,7 +31,8 @@
                                is_timedelta64_dtype, is_datetime64_dtype,
                                is_categorical_dtype, _values_from_object,
                                is_datetime_or_timedelta_dtype, is_bool,
-                               is_bool_dtype, AbstractMethodError)
+                               is_bool_dtype, AbstractMethodError,
+                               _maybe_fill)
 from pandas.core.config import option_context
 import pandas.lib as lib
 from pandas.lib import Timestamp
@@ -1725,14 +1726,15 @@ def _cython_operation(self, kind, values, how, axis):
         labels, _, _ = self.group_info
 
         if kind == 'aggregate':
-            result = np.empty(out_shape, dtype=out_dtype)
-            result.fill(np.nan)
+            result = _maybe_fill(np.empty(out_shape, dtype=out_dtype),
+                                 fill_value=np.nan)
             counts = np.zeros(self.ngroups, dtype=np.int64)
             result = self._aggregate(
                 result, counts, values, labels, func, is_numeric)
         elif kind == 'transform':
-            result = np.empty_like(values, dtype=out_dtype)
-            result.fill(np.nan)
+            result = _maybe_fill(np.empty_like(values, dtype=out_dtype),
+                                 fill_value=np.nan)
+
             # temporary storange for running-total type tranforms
             accum = np.empty(out_shape, dtype=out_dtype)
             result = self._transform(

pandas/core/internals.py

@@ -14,7 +14,7 @@
                                 is_dtype_equal, is_null_datelike_scalar,
                                 _maybe_promote, is_timedelta64_dtype,
                                 is_datetime64_dtype, is_datetimetz, is_sparse,
-                                array_equivalent,
+                                array_equivalent, _is_na_compat,
                                 _maybe_convert_string_to_object,
                                 is_categorical, is_datetimelike_v_numeric,
                                 is_numeric_v_string_like, is_internal_type)
@@ -4392,7 +4392,6 @@ def _putmask_smart(v, m, n):
     m : `mask`, applies to both sides (array like)
     n : `new values` either scalar or an array like aligned with `values`
     """
-
     # n should be the length of the mask or a scalar here
     if not is_list_like(n):
         n = np.array([n] * len(m))
@@ -4403,6 +4402,12 @@ def _putmask_smart(v, m, n):
     # will work in the current dtype
     try:
         nn = n[m]
+
+        # make sure that we have a nullable type
+        # if we have nulls
+        if not _is_na_compat(v, nn[0]):
+            raise ValueError
+
         nn_at = nn.astype(v.dtype)
 
         # avoid invalid dtype comparisons

pandas/tests/test_groupby.py

@@ -5774,7 +5774,6 @@ def test_cython_group_transform_algos(self):
         data = np.array([np.timedelta64(1, 'ns')] * 5, dtype='m8[ns]')[:, None]
         accum = np.array([[0]], dtype='int64')
         actual = np.zeros_like(data, dtype='int64')
-        actual.fill(np.nan)
         pd.algos.group_cumsum(actual, data.view('int64'), labels, accum)
         expected = np.array([np.timedelta64(1, 'ns'), np.timedelta64(
             2, 'ns'), np.timedelta64(3, 'ns'), np.timedelta64(4, 'ns'),