From 7272320f9cd586641c552b34d235b9191869e2b4 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 9 Jan 2019 09:57:54 -0800 Subject: [PATCH 1/9] DataFrame reduction ops (axis=1) with same tz dtypes --- pandas/core/internals/managers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index ab033ff4c1c4b..3e096e4b512c6 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -605,7 +605,13 @@ def _consolidate_check(self): def is_mixed_type(self): # Warning, consolidation needs to get checked upstairs self._consolidate_inplace() - return len(self.blocks) > 1 + if len(self.blocks) == 1: + return False + # If all the blocks are datetimetz and the same timezone, we are + # _not_ mixed type + elif all(block.is_datetimetz for block in self.blocks): + return len({block.dtype for block in self.blocks}) > 1 + return True @property def is_numeric_mixed_type(self): From c295a92da229428d44f7fb26536c87518f474b07 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 9 Jan 2019 22:37:11 -0800 Subject: [PATCH 2/9] change behavior in reduce method --- pandas/core/frame.py | 4 +++- pandas/core/generic.py | 5 +++++ pandas/core/internals/managers.py | 15 ++++++++------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 7bbbdd70e062e..cf50db8fe1c88 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -7390,7 +7390,9 @@ def f(x): return op(x, axis=axis, skipna=skipna, **kwds) # exclude timedelta/datetime unless we are uniform types - if axis == 1 and self._is_mixed_type and self._is_datelike_mixed_type: + if (axis == 1 and self._is_datelike_mixed_type + and (self._is_mixed_type + and not self._is_all_same_datetimetz_types)): numeric_only = True if numeric_only is None: diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 1e6ae71660617..3cd3b994f6a6e 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -5164,6 +5164,11 @@ def _is_datelike_mixed_type(self): f = lambda: self._data.is_datelike_mixed_type return self._protect_consolidate(f) + @property + def _is_all_same_datetimetz_types(self): + f = lambda: self._data.all_same_datetimetz_types + return self._protect_consolidate(f) + def _check_inplace_setting(self, value): """ check whether we allow in-place setting with this type of value """ diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 3e096e4b512c6..c68ec6542dcfb 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -605,13 +605,7 @@ def _consolidate_check(self): def is_mixed_type(self): # Warning, consolidation needs to get checked upstairs self._consolidate_inplace() - if len(self.blocks) == 1: - return False - # If all the blocks are datetimetz and the same timezone, we are - # _not_ mixed type - elif all(block.is_datetimetz for block in self.blocks): - return len({block.dtype for block in self.blocks}) > 1 - return True + return len(self.blocks) > 1 @property def is_numeric_mixed_type(self): @@ -645,6 +639,13 @@ def is_view(self): return False + @property + def all_same_datetimetz_types(self): + """Whether all blocks are datetimetz and the same timezone""" + if all(block.is_datetimetz for block in self.blocks): + return len({block.dtype for block in self.blocks}) == 1 + return False + def get_bool_data(self, copy=False): """ Parameters From 0f3b5c01950e0263b15cab29059b9c5382ffeb9b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 10 Jan 2019 22:29:40 -0800 Subject: [PATCH 3/9] add test --- pandas/tests/frame/test_timezones.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pandas/tests/frame/test_timezones.py b/pandas/tests/frame/test_timezones.py index fd6587c73b8fa..a663e35c758a0 100644 --- a/pandas/tests/frame/test_timezones.py +++ b/pandas/tests/frame/test_timezones.py @@ -196,3 +196,15 @@ def test_tz_localize_convert_copy_inplace_mutate(self, copy, method, tz): index=date_range('20131027', periods=5, freq='1H', tz=tz)) tm.assert_frame_equal(result, expected) + + @pytest.mark.parametrize('op, expected_col', [ + ['max', 'a'], ['min', 'b'] + ]) + def test_same_tz_min_max_axis_1(self, op, expected_col): + # GH 10390 + df = DataFrame(date_range('2016-01-01 00:00:00', periods=3, tz='UTC'), + columns=['a']) + df['b'] = df.a.subtract(pd.Timedelta(seconds=3600)) + result = getattr(df, op)(axis=1) + expected = df[expected_col] + tm.assert_series_equal(result, expected) From f84f126575609cb046d7e1b0bec9978d3a9f1d5b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 13 Jan 2019 16:17:37 -0800 Subject: [PATCH 4/9] Add whatsnew and add short circuting in check --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/core/internals/managers.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index a18739f4b26bf..685bf5b683373 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1647,6 +1647,7 @@ Timezones - Bug in :meth:`DataFrame.any` returns wrong value when ``axis=1`` and the data is of datetimelike type (:issue:`23070`) - Bug in :meth:`DatetimeIndex.to_period` where a timezone aware index was converted to UTC first before creating :class:`PeriodIndex` (:issue:`22905`) - Bug in :meth:`DataFrame.tz_localize`, :meth:`DataFrame.tz_convert`, :meth:`Series.tz_localize`, and :meth:`Series.tz_convert` where ``copy=False`` would mutate the original argument inplace (:issue:`6326`) +- Bug in :meth:`DataFrame.max` and :meth:`DataFrame.min` with ``axis=1`` where a :class:`Series` with ``NaN`` would be returned when all columns contained the same timezone (:issue:`10390`) Offsets ^^^^^^^ diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index c68ec6542dcfb..a622147d0eb77 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -642,9 +642,12 @@ def is_view(self): @property def all_same_datetimetz_types(self): """Whether all blocks are datetimetz and the same timezone""" - if all(block.is_datetimetz for block in self.blocks): - return len({block.dtype for block in self.blocks}) == 1 - return False + dtypes = set() + for block in self.blocks: + if not block.is_datetimetz or len(dtypes) > 1: + return False + dtypes.add(block.dtype) + return True def get_bool_data(self, copy=False): """ From de214c4b0bb71a1509258ef07257c722ea7fd371 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 13 Jan 2019 16:26:42 -0800 Subject: [PATCH 5/9] still check dtypes length --- pandas/core/internals/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index a622147d0eb77..f14b914589fe5 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -647,7 +647,7 @@ def all_same_datetimetz_types(self): if not block.is_datetimetz or len(dtypes) > 1: return False dtypes.add(block.dtype) - return True + return len(dtypes) == 1 def get_bool_data(self, copy=False): """ From a21a7ac7b5386defac46d1e1c5140d5eb701743a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 13 Jan 2019 17:18:40 -0800 Subject: [PATCH 6/9] move test to test_reductions.py --- pandas/tests/frame/test_timezones.py | 12 ------------ pandas/tests/reductions/test_reductions.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pandas/tests/frame/test_timezones.py b/pandas/tests/frame/test_timezones.py index a663e35c758a0..fd6587c73b8fa 100644 --- a/pandas/tests/frame/test_timezones.py +++ b/pandas/tests/frame/test_timezones.py @@ -196,15 +196,3 @@ def test_tz_localize_convert_copy_inplace_mutate(self, copy, method, tz): index=date_range('20131027', periods=5, freq='1H', tz=tz)) tm.assert_frame_equal(result, expected) - - @pytest.mark.parametrize('op, expected_col', [ - ['max', 'a'], ['min', 'b'] - ]) - def test_same_tz_min_max_axis_1(self, op, expected_col): - # GH 10390 - df = DataFrame(date_range('2016-01-01 00:00:00', periods=3, tz='UTC'), - columns=['a']) - df['b'] = df.a.subtract(pd.Timedelta(seconds=3600)) - result = getattr(df, op)(axis=1) - expected = df[expected_col] - tm.assert_series_equal(result, expected) diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index d27308029fa19..b269fca6f9ea6 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -135,6 +135,19 @@ def test_nanops(self): assert obj.argmin(skipna=False) == -1 assert obj.argmax(skipna=False) == -1 + @pytest.mark.parametrize('op, expected_col', [ + ['max', 'a'], ['min', 'b'] + ]) + def test_same_tz_min_max_axis_1(self, op, expected_col): + # GH 10390 + df = DataFrame(pd.date_range('2016-01-01 00:00:00', periods=3, + tz='UTC'), + columns=['a']) + df['b'] = df.a.subtract(pd.Timedelta(seconds=3600)) + result = getattr(df, op)(axis=1) + expected = df[expected_col] + tm.assert_series_equal(result, expected) + class TestSeriesReductions(object): # Note: the name TestSeriesReductions indicates these tests From d8e06039a69679e9fedd4b976a1aa9236993443a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 14 Jan 2019 10:25:18 -0800 Subject: [PATCH 7/9] use _is_homegeneous_type --- pandas/core/frame.py | 4 ++-- pandas/core/generic.py | 5 ----- pandas/core/internals/managers.py | 10 ---------- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index cf50db8fe1c88..c10e15c0ff580 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -7391,8 +7391,8 @@ def f(x): # exclude timedelta/datetime unless we are uniform types if (axis == 1 and self._is_datelike_mixed_type - and (self._is_mixed_type - and not self._is_all_same_datetimetz_types)): + and not (self._is_homogeneous_type + and self._data.blocks[0].is_datetimetz)): numeric_only = True if numeric_only is None: diff --git a/pandas/core/generic.py b/pandas/core/generic.py index b940a5590aa0b..a0ee9cb253fef 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -5164,11 +5164,6 @@ def _is_datelike_mixed_type(self): f = lambda: self._data.is_datelike_mixed_type return self._protect_consolidate(f) - @property - def _is_all_same_datetimetz_types(self): - f = lambda: self._data.all_same_datetimetz_types - return self._protect_consolidate(f) - def _check_inplace_setting(self, value): """ check whether we allow in-place setting with this type of value """ diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index f14b914589fe5..ab033ff4c1c4b 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -639,16 +639,6 @@ def is_view(self): return False - @property - def all_same_datetimetz_types(self): - """Whether all blocks are datetimetz and the same timezone""" - dtypes = set() - for block in self.blocks: - if not block.is_datetimetz or len(dtypes) > 1: - return False - dtypes.add(block.dtype) - return len(dtypes) == 1 - def get_bool_data(self, copy=False): """ Parameters From 4964dab232fb1daa79c527a5cf374a4d97e30562 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 14 Jan 2019 11:55:26 -0800 Subject: [PATCH 8/9] change condidtion --- pandas/core/frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index c10e15c0ff580..2f58aca189c70 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -7391,8 +7391,8 @@ def f(x): # exclude timedelta/datetime unless we are uniform types if (axis == 1 and self._is_datelike_mixed_type - and not (self._is_homogeneous_type - and self._data.blocks[0].is_datetimetz)): + and (not self._is_homogeneous_type + and not self._data.blocks[0].is_datetimetz)): numeric_only = True if numeric_only is None: From 190db02704365346f860fd9f9a024a6592c35bc6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 15 Jan 2019 13:39:10 -0800 Subject: [PATCH 9/9] Use is_datetime64tz_type --- pandas/core/frame.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 2f58aca189c70..309fb3b841461 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -49,6 +49,7 @@ find_common_type) from pandas.core.dtypes.common import ( is_dict_like, + is_datetime64tz_dtype, is_object_dtype, is_extension_type, is_extension_array_dtype, @@ -7392,7 +7393,7 @@ def f(x): # exclude timedelta/datetime unless we are uniform types if (axis == 1 and self._is_datelike_mixed_type and (not self._is_homogeneous_type - and not self._data.blocks[0].is_datetimetz)): + and not is_datetime64tz_dtype(self.dtypes[0]))): numeric_only = True if numeric_only is None: