From 674e724eed5efcfdfaea3addcbb3f4361eef95cb Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 9 Jan 2019 19:56:10 -0800 Subject: [PATCH 1/4] fix interpreting int as second--> nano --- pandas/core/internals/blocks.py | 4 ++-- pandas/tests/series/test_missing.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 70e4f44cb5de8..4059d2e983413 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2499,9 +2499,9 @@ def _can_hold_element(self, element): def fillna(self, value, **kwargs): # allow filling with integers to be - # interpreted as seconds + # interpreted as nanoseconds if is_integer(value) and not isinstance(value, np.timedelta64): - value = Timedelta(value, unit='s') + value = self._box_func(value) return super(TimeDeltaBlock, self).fillna(value, **kwargs) def _try_coerce_args(self, values, other): diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 90ef465c5f239..0defc555a4745 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -17,7 +17,7 @@ import pandas as pd from pandas import ( Categorical, DataFrame, Index, IntervalIndex, MultiIndex, NaT, Series, - Timestamp, date_range, isna) + Timedelta, Timestamp, date_range, isna) from pandas.core.series import remove_na import pandas.util.testing as tm from pandas.util.testing import assert_frame_equal, assert_series_equal @@ -70,9 +70,10 @@ def test_timedelta_fillna(self): timedelta(days=1, seconds=9 * 3600 + 60 + 1)]) assert_series_equal(result, expected) - # interprested as seconds + # interprested as nanoseconds result = td.fillna(1) - expected = Series([timedelta(seconds=1), timedelta(0), timedelta(1), + expected = Series([Timedelta(nanoseconds=1), + timedelta(0), timedelta(1), timedelta(days=1, seconds=9 * 3600 + 60 + 1)]) assert_series_equal(result, expected) From f059530ddff1e1f91d37e3f8c3d293765ce66cfb Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 10 Jan 2019 07:17:34 -0800 Subject: [PATCH 2/4] whatsnew --- doc/source/whatsnew/v0.24.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index aee3d78243d2e..e5c43f52e5a97 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -436,6 +436,7 @@ Backwards incompatible API changes - Incorrectly passing a :class:`DatetimeIndex` to :meth:`MultiIndex.from_tuples`, rather than a sequence of tuples, now raises a ``TypeError`` rather than a ``ValueError`` (:issue:`24024`) - :func:`pd.offsets.generate_range` argument ``time_rule`` has been removed; use ``offset`` instead (:issue:`24157`) - In 0.23.x, pandas would raise a ``ValueError`` on a merge of a numeric column (e.g. ``int`` dtyped column) and an ``object`` dtyped column (:issue:`9780`). We have re-enabled the ability to merge ``object`` and other dtypes; pandas will still raise on a merge between a numeric and an ``object`` dtyped column that is composed only of strings (:issue:`21681`) +- Passing an integer to :method:`Series.fillna` on a ``Series`` with ``timedelta64[ns]`` dtype will now treat that integer as a number of nanoseconds instead of a number of seconds (:issue:`24694`) Percentage change on groupby ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From e99fd757db80bdb15d5d6e4b3a73869fdd33c3e3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 10 Jan 2019 08:23:33 -0800 Subject: [PATCH 3/4] deprecate instead of change --- doc/source/whatsnew/v0.24.0.rst | 2 +- pandas/core/internals/blocks.py | 9 ++++++++- pandas/tests/series/test_missing.py | 18 +++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index e5c43f52e5a97..2e4e3c87c93e6 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -436,7 +436,6 @@ Backwards incompatible API changes - Incorrectly passing a :class:`DatetimeIndex` to :meth:`MultiIndex.from_tuples`, rather than a sequence of tuples, now raises a ``TypeError`` rather than a ``ValueError`` (:issue:`24024`) - :func:`pd.offsets.generate_range` argument ``time_rule`` has been removed; use ``offset`` instead (:issue:`24157`) - In 0.23.x, pandas would raise a ``ValueError`` on a merge of a numeric column (e.g. ``int`` dtyped column) and an ``object`` dtyped column (:issue:`9780`). We have re-enabled the ability to merge ``object`` and other dtypes; pandas will still raise on a merge between a numeric and an ``object`` dtyped column that is composed only of strings (:issue:`21681`) -- Passing an integer to :method:`Series.fillna` on a ``Series`` with ``timedelta64[ns]`` dtype will now treat that integer as a number of nanoseconds instead of a number of seconds (:issue:`24694`) Percentage change on groupby ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1310,6 +1309,7 @@ Deprecations - In :meth:`Series.where` with Categorical data, providing an ``other`` that is not present in the categories is deprecated. Convert the categorical to a different dtype or add the ``other`` to the categories first (:issue:`24077`). - :meth:`Series.clip_lower`, :meth:`Series.clip_upper`, :meth:`DataFrame.clip_lower` and :meth:`DataFrame.clip_upper` are deprecated and will be removed in a future version. Use ``Series.clip(lower=threshold)``, ``Series.clip(upper=threshold)`` and the equivalent ``DataFrame`` methods (:issue:`24203`) - :meth:`Series.nonzero` is deprecated and will be removed in a future version (:issue:`18262`) +- :meth:`Series.fillna` and :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtypes interpretation of integer values as seconds is deprecated. In a future version, integer values will be treated as nanoseconds (:issue:`24694`) .. _whatsnew_0240.deprecations.datetimelike_int_ops: diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 4059d2e983413..2fecfa15181b8 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2501,7 +2501,14 @@ def fillna(self, value, **kwargs): # allow filling with integers to be # interpreted as nanoseconds if is_integer(value) and not isinstance(value, np.timedelta64): - value = self._box_func(value) + # Deprecation GH#24694, GH#19233 + warnings.warn("The interpretation of integers as seconds in " + "fillna is deprecated, will be changed in a future " + "version to be interpreted as nanoseconds. To " + "retain the old behavior, pass " + "`fillna(Timedelta(seconds=value))`", + FutureWarning, stacklevel=6) + value = Timedelta(value, unit='s') return super(TimeDeltaBlock, self).fillna(value, **kwargs) def _try_coerce_args(self, values, other): diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 0defc555a4745..f4f16ff2d3ac1 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -17,7 +17,7 @@ import pandas as pd from pandas import ( Categorical, DataFrame, Index, IntervalIndex, MultiIndex, NaT, Series, - Timedelta, Timestamp, date_range, isna) + Timestamp, date_range, isna) from pandas.core.series import remove_na import pandas.util.testing as tm from pandas.util.testing import assert_frame_equal, assert_series_equal @@ -65,14 +65,16 @@ def test_timedelta_fillna(self): td = s.diff() # reg fillna - result = td.fillna(0) + with tm.assert_produces_warning(FutureWarning): + result = td.fillna(0) expected = Series([timedelta(0), timedelta(0), timedelta(1), timedelta(days=1, seconds=9 * 3600 + 60 + 1)]) assert_series_equal(result, expected) - # interprested as nanoseconds - result = td.fillna(1) - expected = Series([Timedelta(nanoseconds=1), + # interpreted as seconds, deprecated + with tm.assert_produces_warning(FutureWarning): + result = td.fillna(1) + expected = Series([timedelta(seconds=1), timedelta(0), timedelta(1), timedelta(days=1, seconds=9 * 3600 + 60 + 1)]) assert_series_equal(result, expected) @@ -97,14 +99,16 @@ def test_timedelta_fillna(self): # ffill td[2] = np.nan result = td.ffill() - expected = td.fillna(0) + with tm.assert_produces_warning(FutureWarning): + expected = td.fillna(0) expected[0] = np.nan assert_series_equal(result, expected) # bfill td[2] = np.nan result = td.bfill() - expected = td.fillna(0) + with tm.assert_produces_warning(FutureWarning): + expected = td.fillna(0) expected[2] = timedelta(days=1, seconds=9 * 3600 + 60 + 1) assert_series_equal(result, expected) From e9cb0aba7be5b65366ad76a57783fa87b3e57f74 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 10 Jan 2019 08:45:20 -0800 Subject: [PATCH 4/4] Change reinterpration to raising --- doc/source/whatsnew/v0.24.0.rst | 2 +- pandas/core/internals/blocks.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 2e4e3c87c93e6..960b205c49c61 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1309,7 +1309,7 @@ Deprecations - In :meth:`Series.where` with Categorical data, providing an ``other`` that is not present in the categories is deprecated. Convert the categorical to a different dtype or add the ``other`` to the categories first (:issue:`24077`). - :meth:`Series.clip_lower`, :meth:`Series.clip_upper`, :meth:`DataFrame.clip_lower` and :meth:`DataFrame.clip_upper` are deprecated and will be removed in a future version. Use ``Series.clip(lower=threshold)``, ``Series.clip(upper=threshold)`` and the equivalent ``DataFrame`` methods (:issue:`24203`) - :meth:`Series.nonzero` is deprecated and will be removed in a future version (:issue:`18262`) -- :meth:`Series.fillna` and :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtypes interpretation of integer values as seconds is deprecated. In a future version, integer values will be treated as nanoseconds (:issue:`24694`) +- Passing an integer to :meth:`Series.fillna` and :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtypes is deprecated, will raise ``TypeError`` in a future version. Use ``obj.fillna(pd.Timedelta(...))` instead (:issue:`24694`) .. _whatsnew_0240.deprecations.datetimelike_int_ops: diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 2fecfa15181b8..602f4856d5fc0 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2502,11 +2502,10 @@ def fillna(self, value, **kwargs): # interpreted as nanoseconds if is_integer(value) and not isinstance(value, np.timedelta64): # Deprecation GH#24694, GH#19233 - warnings.warn("The interpretation of integers as seconds in " - "fillna is deprecated, will be changed in a future " - "version to be interpreted as nanoseconds. To " - "retain the old behavior, pass " - "`fillna(Timedelta(seconds=value))`", + warnings.warn("Passing integers to fillna is deprecated, will " + "raise a TypeError in a future version. To retain " + "the old behavior, pass pd.Timedelta(seconds=n) " + "instead.", FutureWarning, stacklevel=6) value = Timedelta(value, unit='s') return super(TimeDeltaBlock, self).fillna(value, **kwargs)