From 608003d407e8d4cc337bc8e808c283e8a34589b6 Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 15 Oct 2020 22:36:36 +0200 Subject: [PATCH 1/4] BUG: Fix bug in quantile() for resample and groupby with Timedelta --- doc/source/whatsnew/v1.2.0.rst | 1 + pandas/core/groupby/groupby.py | 4 ++++ pandas/tests/groupby/test_quantile.py | 18 ++++++++++++++++++ pandas/tests/resample/test_timedelta.py | 19 +++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 9fc094330fb36..6f7c0a514ef23 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -462,6 +462,7 @@ Groupby/resample/rolling - Bug in :meth:`RollingGroupby.count` where a ``ValueError`` was raised when specifying the ``closed`` parameter (:issue:`35869`) - Bug in :meth:`DataFrame.groupby.rolling` returning wrong values with partial centered window (:issue:`36040`). - Bug in :meth:`DataFrameGroupBy.rolling` returned wrong values with timeaware window containing ``NaN``. Raises ``ValueError`` because windows are not monotonic now (:issue:`34617`) +- Bug in :meth:`DataFrameGroupBy.quantile` and :meth:`DataFrame.resample().quantile()` raised ``TypeError`` when values to calculate quantile over where ``Timedelta`` (:issue:`29485`) Reshaping ^^^^^^^^^ diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 3938adce6736a..0364092ff5d76 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -60,6 +60,7 @@ class providing the base-class of operations. is_numeric_dtype, is_object_dtype, is_scalar, + is_timedelta64_dtype, ) from pandas.core.dtypes.missing import isna, notna @@ -2151,6 +2152,9 @@ def pre_processor(vals: np.ndarray) -> Tuple[np.ndarray, Optional[Type]]: elif is_datetime64_dtype(vals.dtype): inference = "datetime64[ns]" vals = np.asarray(vals).astype(float) + elif is_timedelta64_dtype(vals.dtype): + inference = "timedelta64[ns]" + vals = np.asarray(vals).astype(float) return vals, inference diff --git a/pandas/tests/groupby/test_quantile.py b/pandas/tests/groupby/test_quantile.py index 9338742195bfe..d56f1fbea456b 100644 --- a/pandas/tests/groupby/test_quantile.py +++ b/pandas/tests/groupby/test_quantile.py @@ -240,3 +240,21 @@ def test_groupby_quantile_skips_invalid_dtype(q): result = df.groupby("a").quantile(q) expected = df.groupby("a")[["b"]].quantile(q) tm.assert_frame_equal(result, expected) + + +def test_groupby_timedelta_quantile(): + # GH: 29485 + df = pd.DataFrame( + {"value": pd.to_timedelta(np.arange(4), unit="s"), "group": [1, 1, 2, 2]} + ) + result = df.groupby("group").quantile(0.99) + expected = pd.DataFrame( + { + "value": [ + pd.Timedelta("0 days 00:00:00.990000"), + pd.Timedelta("0 days 00:00:02.990000"), + ] + }, + index=pd.Index([1, 2], name="group"), + ) + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/resample/test_timedelta.py b/pandas/tests/resample/test_timedelta.py index 3fa85e62d028c..51860a04dce7b 100644 --- a/pandas/tests/resample/test_timedelta.py +++ b/pandas/tests/resample/test_timedelta.py @@ -165,3 +165,22 @@ def test_resample_with_timedelta_yields_no_empty_groups(): index=pd.timedelta_range(start="1s", periods=13, freq="3s"), ) tm.assert_frame_equal(result, expected) + + +def test_resample_quantile_timedelta(): + # GH: 29485 + df = pd.DataFrame( + {"value": pd.to_timedelta(np.arange(4), unit="s")}, + index=pd.date_range("20200101", periods=4, tz="UTC"), + ) + result = df.resample("2D").quantile(0.99) + expected = pd.DataFrame( + { + "value": [ + pd.Timedelta("0 days 00:00:00.990000"), + pd.Timedelta("0 days 00:00:02.990000"), + ] + }, + index=pd.date_range("20200101", periods=2, tz="UTC", freq="2D"), + ) + tm.assert_frame_equal(result, expected) From 9a50db8e62e9a2f1fa13f78740ec9122c40ebfa5 Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 22 Oct 2020 23:45:06 +0200 Subject: [PATCH 2/4] Change whatsnew to reflect link to method --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 6f7c0a514ef23..9958078e6ee8f 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -462,7 +462,7 @@ Groupby/resample/rolling - Bug in :meth:`RollingGroupby.count` where a ``ValueError`` was raised when specifying the ``closed`` parameter (:issue:`35869`) - Bug in :meth:`DataFrame.groupby.rolling` returning wrong values with partial centered window (:issue:`36040`). - Bug in :meth:`DataFrameGroupBy.rolling` returned wrong values with timeaware window containing ``NaN``. Raises ``ValueError`` because windows are not monotonic now (:issue:`34617`) -- Bug in :meth:`DataFrameGroupBy.quantile` and :meth:`DataFrame.resample().quantile()` raised ``TypeError`` when values to calculate quantile over where ``Timedelta`` (:issue:`29485`) +- Bug in :meth:`df.groupby(..).quantile() ` and :meth:`df.resample(..).quantile() ` raised ``TypeError`` when values to calculate quantile over where ``Timedelta`` (:issue:`29485`) Reshaping ^^^^^^^^^ From 6bd6e137864657d94f1f86ccf90285fba8286c81 Mon Sep 17 00:00:00 2001 From: phofl Date: Fri, 23 Oct 2020 00:04:42 +0200 Subject: [PATCH 3/4] Fix pattern --- pandas/tests/groupby/test_quantile.py | 4 ++-- pandas/tests/resample/test_timedelta.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/groupby/test_quantile.py b/pandas/tests/groupby/test_quantile.py index dfd46bd783aea..e48f10ebacb79 100644 --- a/pandas/tests/groupby/test_quantile.py +++ b/pandas/tests/groupby/test_quantile.py @@ -240,11 +240,11 @@ def test_groupby_quantile_skips_invalid_dtype(q): def test_groupby_timedelta_quantile(): # GH: 29485 - df = pd.DataFrame( + df = DataFrame( {"value": pd.to_timedelta(np.arange(4), unit="s"), "group": [1, 1, 2, 2]} ) result = df.groupby("group").quantile(0.99) - expected = pd.DataFrame( + expected = DataFrame( { "value": [ pd.Timedelta("0 days 00:00:00.990000"), diff --git a/pandas/tests/resample/test_timedelta.py b/pandas/tests/resample/test_timedelta.py index 89b388e8f4e62..1c440b889b146 100644 --- a/pandas/tests/resample/test_timedelta.py +++ b/pandas/tests/resample/test_timedelta.py @@ -169,12 +169,12 @@ def test_resample_with_timedelta_yields_no_empty_groups(): def test_resample_quantile_timedelta(): # GH: 29485 - df = pd.DataFrame( + df = DataFrame( {"value": pd.to_timedelta(np.arange(4), unit="s")}, index=pd.date_range("20200101", periods=4, tz="UTC"), ) result = df.resample("2D").quantile(0.99) - expected = pd.DataFrame( + expected = DataFrame( { "value": [ pd.Timedelta("0 days 00:00:00.990000"), From 187b668bd00b4dd8c19304b18b658d5d63d5888a Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 27 Oct 2020 22:30:48 +0100 Subject: [PATCH 4/4] Change whatsnew --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 22bf8bb89fc13..2d86b8b274605 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -495,7 +495,7 @@ Groupby/resample/rolling - Bug in :meth:`DataFrame.groupby.rolling` returning wrong values with partial centered window (:issue:`36040`). - Bug in :meth:`DataFrameGroupBy.rolling` returned wrong values with timeaware window containing ``NaN``. Raises ``ValueError`` because windows are not monotonic now (:issue:`34617`) - Bug in :meth:`Rolling.__iter__` where a ``ValueError`` was not raised when ``min_periods`` was larger than ``window`` (:issue:`37156`) -- Bug in :meth:`df.groupby(..).quantile() ` and :meth:`df.resample(..).quantile() ` raised ``TypeError`` when values to calculate quantile over where ``Timedelta`` (:issue:`29485`) +- Bug in :meth:`df.groupby(..).quantile() ` and :meth:`df.resample(..).quantile() ` raised ``TypeError`` when values were of type ``Timedelta`` (:issue:`29485`) Reshaping ^^^^^^^^^