From 1fd4c2a85274d9eba08538830f40b00d94c310a2 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 4 Jan 2019 23:33:27 -0800 Subject: [PATCH 1/6] TST: Fixed timezone issues post DatetimeArray --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/tests/frame/test_reshape.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 3be87c4cabaf0..970d185a51da2 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1694,6 +1694,7 @@ Reshaping - Constructing a DataFrame with an index argument that wasn't already an instance of :class:`~pandas.core.Index` was broken (:issue:`22227`). - Bug in :class:`DataFrame` prevented list subclasses to be used to construction (:issue:`21226`) - Bug in :func:`DataFrame.unstack` and :func:`DataFrame.pivot_table` returning a missleading error message when the resulting DataFrame has more elements than int32 can handle. Now, the error message is improved, pointing towards the actual problem (:issue:`20601`) +- Bug in :func:`DataFrame.unstack` raising a ``ValueError`` when unstacking timezone aware values (:issues:`18338`) .. _whatsnew_0240.bug_fixes.sparse: diff --git a/pandas/tests/frame/test_reshape.py b/pandas/tests/frame/test_reshape.py index 362650714418f..75f40f9edc332 100644 --- a/pandas/tests/frame/test_reshape.py +++ b/pandas/tests/frame/test_reshape.py @@ -936,3 +936,24 @@ def test_unstack_fill_frame_object(): index=list('xyz') ) assert_frame_equal(result, expected) + + +def test_unstack_timezone_aware_values(): + # GH 18338 + df = pd.DataFrame({ + 'timestamp': [ + pd.Timestamp('2017-08-27 01:00:00.709949+0000', tz='UTC')], + 'a': ['a'], + 'b': ['b'], + 'c': ['c'], + }) + result = df.set_index(['a', 'b']).unstack() + expected = pd.DataFrame([[pd.Timestamp('2017-08-27 01:00:00.709949+0000', + tz='UTC'), + 'c']], + index=pd.Index(['a'], name='a'), + columns=pd.MultiIndex( + levels=[['timestamp', 'c'], ['b']], + codes=[[0, 1], [0, 0]], + names=[None, 'b'])) + assert_frame_equal(result, expected) From 3867d4c62b047d6ba56c5f649b539ace92af4c2b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 4 Jan 2019 23:45:16 -0800 Subject: [PATCH 2/6] test DataFrame.stack with tz data --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/tests/frame/test_reshape.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 970d185a51da2..2adbdf6170a76 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1695,6 +1695,7 @@ Reshaping - Bug in :class:`DataFrame` prevented list subclasses to be used to construction (:issue:`21226`) - Bug in :func:`DataFrame.unstack` and :func:`DataFrame.pivot_table` returning a missleading error message when the resulting DataFrame has more elements than int32 can handle. Now, the error message is improved, pointing towards the actual problem (:issue:`20601`) - Bug in :func:`DataFrame.unstack` raising a ``ValueError`` when unstacking timezone aware values (:issues:`18338`) +- Bug in :func:`DataFrame.stack` where timezone aware values were converted to timezone naive values (:issues:`19420`) .. _whatsnew_0240.bug_fixes.sparse: diff --git a/pandas/tests/frame/test_reshape.py b/pandas/tests/frame/test_reshape.py index 75f40f9edc332..9003084b2eab3 100644 --- a/pandas/tests/frame/test_reshape.py +++ b/pandas/tests/frame/test_reshape.py @@ -957,3 +957,15 @@ def test_unstack_timezone_aware_values(): codes=[[0, 1], [0, 0]], names=[None, 'b'])) assert_frame_equal(result, expected) + + +def test_stack_timezone_aware_values(): + # GH 19420 + ts = pd.date_range(freq="D", start="20180101", end="20180103", + tz="America/New_York") + df = pd.DataFrame({"A": ts}, index=["a", "b", "c"]) + result = df.stack() + expected = pd.Series(ts, + index=pd.MultiIndex(levels=[['a', 'b', 'c'], ['A']], + codes=[[0, 1, 2], [0, 0, 0]])) + assert_series_equal(result, expected) From ad8ab5f542626e7c018236c2c62515af623958e8 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 4 Jan 2019 23:54:04 -0800 Subject: [PATCH 3/6] test merge_asof with by_col tz-aware --- doc/source/whatsnew/v0.24.0.rst | 5 +++-- pandas/tests/reshape/merge/test_merge_asof.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 2adbdf6170a76..41ad86e946866 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1694,8 +1694,9 @@ Reshaping - Constructing a DataFrame with an index argument that wasn't already an instance of :class:`~pandas.core.Index` was broken (:issue:`22227`). - Bug in :class:`DataFrame` prevented list subclasses to be used to construction (:issue:`21226`) - Bug in :func:`DataFrame.unstack` and :func:`DataFrame.pivot_table` returning a missleading error message when the resulting DataFrame has more elements than int32 can handle. Now, the error message is improved, pointing towards the actual problem (:issue:`20601`) -- Bug in :func:`DataFrame.unstack` raising a ``ValueError`` when unstacking timezone aware values (:issues:`18338`) -- Bug in :func:`DataFrame.stack` where timezone aware values were converted to timezone naive values (:issues:`19420`) +- Bug in :func:`DataFrame.unstack` where a ``ValueError`` was raised when unstacking timezone aware values (:issue:`18338`) +- Bug in :func:`DataFrame.stack` where timezone aware values were converted to timezone naive values (:issue:`19420`) +- Bug in :func:`merge_asof` where a ``TypeError`` was raised when ``by_col`` were timezone aware values (:issue:`21184`) .. _whatsnew_0240.bug_fixes.sparse: diff --git a/pandas/tests/reshape/merge/test_merge_asof.py b/pandas/tests/reshape/merge/test_merge_asof.py index 1483654daa99e..f3400748a777c 100644 --- a/pandas/tests/reshape/merge/test_merge_asof.py +++ b/pandas/tests/reshape/merge/test_merge_asof.py @@ -1022,3 +1022,19 @@ def test_merge_on_nans(self, func, side): merge_asof(df_null, df, on='a') else: merge_asof(df, df_null, on='a') + + def test_merge_by_col_tz_aware(self): + # GH 21184 + left = pd.DataFrame( + {'by_col': pd.DatetimeIndex(['2018-01-01']).tz_localize('UTC'), + 'on_col': [2], 'values': ['a']}) + right = pd.DataFrame( + {'by_col': pd.DatetimeIndex(['2018-01-01']).tz_localize('UTC'), + 'on_col': [1], 'values': ['b']}) + result = pd.merge_asof(left, right, by='by_col', on='on_col') + expected = pd.DataFrame([ + [pd.Timestamp('2018-01-01', tz='UTC'), 2, 'a', 'b'] + ], + columns=['by_col', 'on_col', 'values_x', + 'values_y']) + assert_frame_equal(result, expected) From 89f87ec5255e111535ea89f616e968e997607f4a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sat, 5 Jan 2019 00:07:13 -0800 Subject: [PATCH 4/6] DatetimeIndex.to_period converting to UTC --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/tests/indexes/datetimes/test_astype.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 41ad86e946866..4ce7ce7b865ea 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1448,6 +1448,7 @@ Timezones - Bug in :func:`to_datetime` where ``utc=True`` was not respected when specifying a ``unit`` and ``errors='ignore'`` (:issue:`23758`) - Bug in :func:`to_datetime` where ``utc=True`` was not respected when passing a :class:`Timestamp` (:issue:`24415`) - Bug in :meth:`DataFrame.any` returns wrong value when ``axis=1`` and the data is of datetimelike type (:issue:`23070`) +- Bug in :meth:`DatetimeIndex.to_period` where a timezone aware index was converted to UTC first before creating :class:`PeriodIndex` (:issue:`22905`) Offsets ^^^^^^^ diff --git a/pandas/tests/indexes/datetimes/test_astype.py b/pandas/tests/indexes/datetimes/test_astype.py index 562be4cf85864..422525255c365 100644 --- a/pandas/tests/indexes/datetimes/test_astype.py +++ b/pandas/tests/indexes/datetimes/test_astype.py @@ -293,6 +293,15 @@ def test_to_period_tz(self, tz): tm.assert_index_equal(result, expected) + @pytest.mark.parametrize('tz', ['Etc/GMT-1', 'Etc/GMT+1']) + def test_to_period_tz_utc_offset_consistency(self, tz): + # GH 22905 + ts = pd.date_range('1/1/2000', '2/1/2000', tz='Etc/GMT-1') + with tm.assert_produces_warning(UserWarning): + result = ts.to_period()[0] + expected = ts[0].to_period() + assert result == expected + def test_to_period_nofreq(self): idx = DatetimeIndex(['2000-01-01', '2000-01-02', '2000-01-04']) with pytest.raises(ValueError): From f1b865630979a2ef365d3ddf8665b5ab79e7f91f Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sat, 5 Jan 2019 00:23:44 -0800 Subject: [PATCH 5/6] lint --- pandas/tests/reshape/merge/test_merge_asof.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/tests/reshape/merge/test_merge_asof.py b/pandas/tests/reshape/merge/test_merge_asof.py index f3400748a777c..1d1d7d48adaab 100644 --- a/pandas/tests/reshape/merge/test_merge_asof.py +++ b/pandas/tests/reshape/merge/test_merge_asof.py @@ -1034,7 +1034,5 @@ def test_merge_by_col_tz_aware(self): result = pd.merge_asof(left, right, by='by_col', on='on_col') expected = pd.DataFrame([ [pd.Timestamp('2018-01-01', tz='UTC'), 2, 'a', 'b'] - ], - columns=['by_col', 'on_col', 'values_x', - 'values_y']) + ], columns=['by_col', 'on_col', 'values_x', 'values_y']) assert_frame_equal(result, expected) From d660481b68f63cab60c134b05a67930f2057ed0a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sat, 5 Jan 2019 15:20:41 -0800 Subject: [PATCH 6/6] Add columns for py2 dict ordering --- pandas/tests/frame/test_reshape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_reshape.py b/pandas/tests/frame/test_reshape.py index 9003084b2eab3..f2f6944a21e03 100644 --- a/pandas/tests/frame/test_reshape.py +++ b/pandas/tests/frame/test_reshape.py @@ -946,7 +946,7 @@ def test_unstack_timezone_aware_values(): 'a': ['a'], 'b': ['b'], 'c': ['c'], - }) + }, columns=['timestamp', 'a', 'b', 'c']) result = df.set_index(['a', 'b']).unstack() expected = pd.DataFrame([[pd.Timestamp('2017-08-27 01:00:00.709949+0000', tz='UTC'),