Merge pull request #5368 from jreback/dtype_conv

BUG: downcasting is now more robust (related GH5174)

Author: jreback

pandas/core/common.py

@@ -1022,6 +1022,7 @@ def _possibly_downcast_to_dtype(result, dtype):
     if np.isscalar(result) or not len(result):
         return result
 
+    trans = lambda x: x
     if isinstance(dtype, compat.string_types):
         if dtype == 'infer':
             inferred_type = lib.infer_dtype(_ensure_object(result.ravel()))
@@ -1037,6 +1038,7 @@ def _possibly_downcast_to_dtype(result, dtype):
             # try to upcast here
             elif inferred_type == 'floating':
                 dtype = 'int64'
+                trans = lambda x: x.round()
 
             else:
                 dtype = 'object'
@@ -1058,15 +1060,15 @@ def _possibly_downcast_to_dtype(result, dtype):
             # do a test on the first element, if it fails then we are done
             r = result.ravel()
             arr = np.array([ r[0] ])
-            if not np.allclose(arr,arr.astype(dtype)):
+            if not np.allclose(arr,trans(arr).astype(dtype)):
                 return result
 
             # a comparable, e.g. a Decimal may slip in here
             elif not isinstance(r[0], (np.integer,np.floating,np.bool,int,float,bool)):
                 return result
 
             if issubclass(result.dtype.type, (np.object_,np.number)) and notnull(result).all():
-                new_result = result.astype(dtype)
+                new_result = trans(result).astype(dtype)
                 try:
                     if np.allclose(new_result,result):
                         return new_result

pandas/tests/test_common.py

@@ -118,6 +118,22 @@ def test_isnull_datetime():
     assert(mask[0])
     assert(not mask[1:].any())
 
+def test_downcast_conv():
+    # test downcasting
+
+    arr = np.array([8.5, 8.6, 8.7, 8.8, 8.9999999999995])
+    result = com._possibly_downcast_to_dtype(arr, 'infer')
+    assert (np.array_equal(result, arr))
+
+    arr = np.array([8., 8., 8., 8., 8.9999999999995])
+    result = com._possibly_downcast_to_dtype(arr, 'infer')
+    expected = np.array([8, 8, 8, 8, 9])
+    assert (np.array_equal(result, expected))
+
+    arr = np.array([8., 8., 8., 8., 9.0000000000005])
+    result = com._possibly_downcast_to_dtype(arr, 'infer')
+    expected = np.array([8, 8, 8, 8, 9])
+    assert (np.array_equal(result, expected))
 
 def test_datetimeindex_from_empty_datetime64_array():
     for unit in [ 'ms', 'us', 'ns' ]:

pandas/tests/test_generic.py

@@ -478,9 +478,9 @@ def test_nan_str_index(self):
     def test_interp_quad(self):
         _skip_if_no_scipy()
         sq = Series([1, 4, np.nan, 16], index=[1, 2, 3, 4])
-        result = sq.interpolate(method='quadratic', downcast=False)
-        expected = Series([1., 4, 9, 16], index=[1, 2, 3, 4])
-        assert_series_equal(result, expected, check_less_precise=True)
+        result = sq.interpolate(method='quadratic')
+        expected = Series([1, 4, 9, 16], index=[1, 2, 3, 4])
+        assert_series_equal(result, expected)
 
     def test_interp_scipy_basic(self):
         _skip_if_no_scipy()
@@ -659,7 +659,7 @@ def test_interp_alt_scipy(self):
         expected = df.copy()
         expected['A'].iloc[2] = 3
         expected['A'].iloc[5] = 6
-        assert_frame_equal(result, expected)
+        assert_frame_equal(result, expected.astype(np.int64))
 
         result = df.interpolate(method='krogh')
         expectedk = df.copy()