diff --git a/doc/source/release.rst b/doc/source/release.rst index 5c7d090becf83..b161d1f785114 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -225,6 +225,7 @@ Bug Fixes - Bug in indexing: empty list lookup caused ``IndexError`` exceptions (:issue:`6536`, :issue:`6551`) - Series.quantile raising on an ``object`` dtype (:issue:`6555`) - Bug in ``.xs`` with a ``nan`` in level when dropped (:issue:`6574`) +- Bug in fillna with method = 'bfill/ffill' and ``datetime64[ns]`` dtype (:issue:`6587`) pandas 0.13.1 ------------- diff --git a/pandas/core/common.py b/pandas/core/common.py index eb3c159ae916d..60a533db01f7f 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -1244,13 +1244,14 @@ def wrapper(arr, mask, limit=None): np.int64) -def pad_1d(values, limit=None, mask=None): +def pad_1d(values, limit=None, mask=None, dtype=None): - dtype = values.dtype.name + if dtype is None: + dtype = values.dtype _method = None if is_float_dtype(values): - _method = getattr(algos, 'pad_inplace_%s' % dtype, None) - elif is_datetime64_dtype(values): + _method = getattr(algos, 'pad_inplace_%s' % dtype.name, None) + elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): _method = _pad_1d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -1259,7 +1260,7 @@ def pad_1d(values, limit=None, mask=None): _method = algos.pad_inplace_object if _method is None: - raise ValueError('Invalid dtype for pad_1d [%s]' % dtype) + raise ValueError('Invalid dtype for pad_1d [%s]' % dtype.name) if mask is None: mask = isnull(values) @@ -1268,13 +1269,14 @@ def pad_1d(values, limit=None, mask=None): return values -def backfill_1d(values, limit=None, mask=None): +def backfill_1d(values, limit=None, mask=None, dtype=None): - dtype = values.dtype.name + if dtype is None: + dtype = values.dtype _method = None if is_float_dtype(values): - _method = getattr(algos, 'backfill_inplace_%s' % dtype, None) - elif is_datetime64_dtype(values): + _method = getattr(algos, 'backfill_inplace_%s' % dtype.name, None) + elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): _method = _backfill_1d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -1283,7 +1285,7 @@ def backfill_1d(values, limit=None, mask=None): _method = algos.backfill_inplace_object if _method is None: - raise ValueError('Invalid dtype for backfill_1d [%s]' % dtype) + raise ValueError('Invalid dtype for backfill_1d [%s]' % dtype.name) if mask is None: mask = isnull(values) @@ -1293,13 +1295,14 @@ def backfill_1d(values, limit=None, mask=None): return values -def pad_2d(values, limit=None, mask=None): +def pad_2d(values, limit=None, mask=None, dtype=None): - dtype = values.dtype.name + if dtype is None: + dtype = values.dtype _method = None if is_float_dtype(values): - _method = getattr(algos, 'pad_2d_inplace_%s' % dtype, None) - elif is_datetime64_dtype(values): + _method = getattr(algos, 'pad_2d_inplace_%s' % dtype.name, None) + elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): _method = _pad_2d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -1308,7 +1311,7 @@ def pad_2d(values, limit=None, mask=None): _method = algos.pad_2d_inplace_object if _method is None: - raise ValueError('Invalid dtype for pad_2d [%s]' % dtype) + raise ValueError('Invalid dtype for pad_2d [%s]' % dtype.name) if mask is None: mask = isnull(values) @@ -1322,13 +1325,14 @@ def pad_2d(values, limit=None, mask=None): return values -def backfill_2d(values, limit=None, mask=None): +def backfill_2d(values, limit=None, mask=None, dtype=None): - dtype = values.dtype.name + if dtype is None: + dtype = values.dtype _method = None if is_float_dtype(values): - _method = getattr(algos, 'backfill_2d_inplace_%s' % dtype, None) - elif is_datetime64_dtype(values): + _method = getattr(algos, 'backfill_2d_inplace_%s' % dtype.name, None) + elif dtype in _DATELIKE_DTYPES or is_datetime64_dtype(values): _method = _backfill_2d_datetime elif is_integer_dtype(values): values = _ensure_float64(values) @@ -1337,7 +1341,7 @@ def backfill_2d(values, limit=None, mask=None): _method = algos.backfill_2d_inplace_object if _method is None: - raise ValueError('Invalid dtype for backfill_2d [%s]' % dtype) + raise ValueError('Invalid dtype for backfill_2d [%s]' % dtype.name) if mask is None: mask = isnull(values) @@ -1503,7 +1507,7 @@ def _interpolate_scipy_wrapper(x, y, new_x, method, fill_value=None, return new_y -def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None): +def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None, dtype=None): """ perform an actual interpolation of values, values will be make 2-d if needed fills inplace, returns the result """ @@ -1525,9 +1529,9 @@ def interpolate_2d(values, method='pad', axis=0, limit=None, fill_value=None): method = _clean_fill_method(method) if method == 'pad': - values = transf(pad_2d(transf(values), limit=limit, mask=mask)) + values = transf(pad_2d(transf(values), limit=limit, mask=mask, dtype=dtype)) else: - values = transf(backfill_2d(transf(values), limit=limit, mask=mask)) + values = transf(backfill_2d(transf(values), limit=limit, mask=mask, dtype=dtype)) # reshape back if ndim == 1: diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 39eb03eebdb8c..2a4ee4dcf8cf6 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -878,7 +878,12 @@ def _interpolate_with_fill(self, method='pad', axis=0, inplace=False, fill_value = self._try_fill(fill_value) values = self.values if inplace else self.values.copy() values = self._try_operate(values) - values = com.interpolate_2d(values, method, axis, limit, fill_value) + values = com.interpolate_2d(values, + method=method, + axis=axis, + limit=limit, + fill_value=fill_value, + dtype=self.dtype) values = self._try_coerce_result(values) blocks = [make_block(values, self.items, self.ref_items, diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py index 4b0af8d0cbdd2..ca1b23ee26da4 100644 --- a/pandas/tests/test_series.py +++ b/pandas/tests/test_series.py @@ -2854,6 +2854,14 @@ def test_datetime64_fillna(self): Timestamp('20130103 9:01:01')]) assert_series_equal(result, expected) + # GH 6587 + # make sure that we are treating as integer when filling + s = Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.000001']) + expected = Series(['2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001'], dtype='M8[ns]') + result = s.fillna(method='backfill') + assert_series_equal(result, expected) + + def test_fillna_int(self): s = Series(np.random.randint(-100, 100, 50)) s.fillna(method='ffill', inplace=True)