From 18f0e4afbbf2adee4712edd8f525daac2cb618ab Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Sun, 3 Feb 2019 00:34:38 -0800 Subject: [PATCH 1/4] Fixing regression in `DataFrame.all` and `DataFrame.any` * `bool_only` parameter is supported again * Commit 36ab8c95a41bbadc286a5a520883b149c86931bd created this regression due to a bug in all/any (pandas-dev/pandas#23070) * Reverted the regression and fixed the bug with a condition * Added tests for `bool_only` parameter --- doc/source/whatsnew/v0.24.1.rst | 1 + pandas/core/frame.py | 3 ++- pandas/tests/frame/test_analytics.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.1.rst b/doc/source/whatsnew/v0.24.1.rst index bc45a14666ba2..9f825a579a54e 100644 --- a/doc/source/whatsnew/v0.24.1.rst +++ b/doc/source/whatsnew/v0.24.1.rst @@ -56,6 +56,7 @@ Fixed Regressions - Fixed regression in :class:`Index.intersection` incorrectly sorting the values by default (:issue:`24959`). - Fixed regression in :func:`merge` when merging an empty ``DataFrame`` with multiple timezone-aware columns on one of the timezone-aware columns (:issue:`25014`). - Fixed regression in :meth:`Series.rename_axis` and :meth:`DataFrame.rename_axis` where passing ``None`` failed to remove the axis name (:issue:`25034`) +- Fixed regression in :meth:`DataFrame.all` and :meth:`DataFrame.any` where ``bool_only`` as ``True`` was ignored (:issue:`25101`) **Timedelta** diff --git a/pandas/core/frame.py b/pandas/core/frame.py index ade05ab27093e..afe09df8b13ab 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -7476,7 +7476,8 @@ def f(x): if filter_type is None or filter_type == 'numeric': data = self._get_numeric_data() elif filter_type == 'bool': - data = self + # GH 25101, # GH 24434 + data = self._get_bool_data() if axis == 0 else self else: # pragma: no cover msg = ("Generating numeric_only data with filter_type {f}" "not supported.".format(f=filter_type)) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 386e5f57617cf..8a0e4f0e83326 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1442,6 +1442,26 @@ def test_any_datetime(self): expected = Series([True, True, True, False]) tm.assert_series_equal(result, expected) + def test_any_all_bool_only(self): + + # GH 25101 + df = DataFrame({"col1": [1,2,3], + "col2": [4,5,6], + "col3": [None, None, None]}) + + result = df.all(bool_only=True) + expected = Series() + tm.assert_equal(result, expected) + + df = DataFrame({"col1": [1,2,3], + "col2": [4,5,6], + "col3": [None, None, None], + "col4":[False, False, True]} + + result = df.all(bool_only=True) + expected = Series() + tm.assert_equal(result, expected) + @pytest.mark.parametrize('func, data, expected', [ (np.any, {}, False), (np.all, {}, True), From 85da8149b60d2125a5e6a4356b7f4f0488bfa9bb Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Sun, 3 Feb 2019 00:40:54 -0800 Subject: [PATCH 2/4] Lint --- pandas/tests/frame/test_analytics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 8a0e4f0e83326..9e4fb96cfa9c0 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1445,18 +1445,18 @@ def test_any_datetime(self): def test_any_all_bool_only(self): # GH 25101 - df = DataFrame({"col1": [1,2,3], - "col2": [4,5,6], + df = DataFrame({"col1": [1, 2, 3], + "col2": [4, 5, 6], "col3": [None, None, None]}) result = df.all(bool_only=True) expected = Series() tm.assert_equal(result, expected) - df = DataFrame({"col1": [1,2,3], - "col2": [4,5,6], + df = DataFrame({"col1": [1, 2, 3], + "col2": [4, 5, 6], "col3": [None, None, None], - "col4":[False, False, True]} + "col4": [False, False, True]}) result = df.all(bool_only=True) expected = Series() From 9e87d5283a5086b3346fadd2221f8ae5a76975e8 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Sun, 3 Feb 2019 00:44:26 -0800 Subject: [PATCH 3/4] Fix test code --- pandas/tests/frame/test_analytics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 9e4fb96cfa9c0..c0f4fb3b4d36e 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1450,7 +1450,7 @@ def test_any_all_bool_only(self): "col3": [None, None, None]}) result = df.all(bool_only=True) - expected = Series() + expected = Series(dtype=np.bool) tm.assert_equal(result, expected) df = DataFrame({"col1": [1, 2, 3], @@ -1459,7 +1459,7 @@ def test_any_all_bool_only(self): "col4": [False, False, True]}) result = df.all(bool_only=True) - expected = Series() + expected = Series({"col4": False}) tm.assert_equal(result, expected) @pytest.mark.parametrize('func, data, expected', [ From 2d6f6f24c57b40aa96c53cfcdc8f9595b286f5a8 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Sun, 3 Feb 2019 10:10:12 -0800 Subject: [PATCH 4/4] Address @jreback comments --- doc/source/whatsnew/v0.24.1.rst | 1 - doc/source/whatsnew/v0.24.2.rst | 4 +--- pandas/tests/frame/test_analytics.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.24.1.rst b/doc/source/whatsnew/v0.24.1.rst index a94d0fc0aa55d..2e4d96dcf9f11 100644 --- a/doc/source/whatsnew/v0.24.1.rst +++ b/doc/source/whatsnew/v0.24.1.rst @@ -58,7 +58,6 @@ Fixed Regressions - Fixed regression in :func:`merge` when merging an empty ``DataFrame`` with multiple timezone-aware columns on one of the timezone-aware columns (:issue:`25014`). - Fixed regression in :meth:`Series.rename_axis` and :meth:`DataFrame.rename_axis` where passing ``None`` failed to remove the axis name (:issue:`25034`) - Fixed regression in :func:`to_timedelta` with `box=False` incorrectly returning a ``datetime64`` object instead of a ``timedelta64`` object (:issue:`24961`) -- Fixed regression in :meth:`DataFrame.all` and :meth:`DataFrame.any` where ``bool_only`` as ``True`` was ignored (:issue:`25101`) .. _whatsnew_0241.bug_fixes: diff --git a/doc/source/whatsnew/v0.24.2.rst b/doc/source/whatsnew/v0.24.2.rst index cba21ce7ee1e6..dc6dc5f6ad57c 100644 --- a/doc/source/whatsnew/v0.24.2.rst +++ b/doc/source/whatsnew/v0.24.2.rst @@ -20,9 +20,7 @@ including other versions of pandas. Fixed Regressions ^^^^^^^^^^^^^^^^^ -- -- -- +- Fixed regression in :meth:`DataFrame.all` and :meth:`DataFrame.any` where ``bool_only=True`` was ignored (:issue:`25101`) .. _whatsnew_0242.enhancements: diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index c0f4fb3b4d36e..456af34e74956 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1451,7 +1451,7 @@ def test_any_all_bool_only(self): result = df.all(bool_only=True) expected = Series(dtype=np.bool) - tm.assert_equal(result, expected) + tm.assert_series_equal(result, expected) df = DataFrame({"col1": [1, 2, 3], "col2": [4, 5, 6], @@ -1460,7 +1460,7 @@ def test_any_all_bool_only(self): result = df.all(bool_only=True) expected = Series({"col4": False}) - tm.assert_equal(result, expected) + tm.assert_series_equal(result, expected) @pytest.mark.parametrize('func, data, expected', [ (np.any, {}, False),