BUG: GH15426 timezone lost in groupby-agg with cython functions

closes #15426

Author: Stephen Rauch <[email protected]>

Closes #15433 from stephenrauch/tz-lost-in-groupby-agg and squashes the following commits:

64a84ca [Stephen Rauch] BUG: GH15426 timezone lost in groupby-agg with cython functions

Author: stephenrauch

doc/source/whatsnew/v0.20.0.txt

@@ -622,6 +622,7 @@ Bug Fixes
 
 - Bug in ``DataFrame.to_stata()`` and ``StataWriter`` which produces incorrectly formatted files to be produced for some locales (:issue:`13856`)
 - Bug in ``pd.concat()`` in which concatting with an empty dataframe with ``join='inner'`` was being improperly handled (:issue:`15328`)
+- Bug in ``groupby.agg()`` incorrectly localizing timezone on ``datetime`` (:issue:`15426`, :issue:`10668`)
 
 
 

pandas/tests/groupby/test_aggregate.py

@@ -6,7 +6,7 @@
 """
 
 from __future__ import print_function
-from datetime import datetime
+from datetime import datetime, timedelta
 from functools import partial
 
 import numpy as np
@@ -738,3 +738,32 @@ def test_agg_over_numpy_arrays(self):
                                 columns=expected_column)
 
         assert_frame_equal(result, expected)
+
+    def test_agg_timezone_round_trip(self):
+        # GH 15426
+        ts = pd.Timestamp("2016-01-01 12:00:00", tz='US/Pacific')
+        df = pd.DataFrame({'a': 1, 'b': [ts + timedelta(minutes=nn)
+                                         for nn in range(10)]})
+
+        result1 = df.groupby('a')['b'].agg(np.min).iloc[0]
+        result2 = df.groupby('a')['b'].agg(lambda x: np.min(x)).iloc[0]
+        result3 = df.groupby('a')['b'].min().iloc[0]
+
+        assert result1 == ts
+        assert result2 == ts
+        assert result3 == ts
+
+        dates = [pd.Timestamp("2016-01-0%d 12:00:00" % i, tz='US/Pacific')
+                 for i in range(1, 5)]
+        df = pd.DataFrame({'A': ['a', 'b'] * 2, 'B': dates})
+        grouped = df.groupby('A')
+
+        ts = df['B'].iloc[0]
+        assert ts == grouped.nth(0)['B'].iloc[0]
+        assert ts == grouped.head(1)['B'].iloc[0]
+        assert ts == grouped.first()['B'].iloc[0]
+        assert ts == grouped.apply(lambda x: x.iloc[0])[0]
+
+        ts = df['B'].iloc[2]
+        assert ts == grouped.last()['B'].iloc[0]
+        assert ts == grouped.apply(lambda x: x.iloc[-1])[0]

pandas/tests/types/test_cast.py

@@ -8,7 +8,7 @@
 from datetime import datetime
 import numpy as np
 
-from pandas import Timedelta, Timestamp
+from pandas import Timedelta, Timestamp, DatetimeIndex
 from pandas.types.cast import (_possibly_downcast_to_dtype,
                                _possibly_convert_objects,
                                _infer_dtype_from_scalar,
@@ -71,6 +71,16 @@ def test_datetimelikes_nan(self):
         res = _possibly_downcast_to_dtype(arr, 'timedelta64[ns]')
         tm.assert_numpy_array_equal(res, exp)
 
+    def test_datetime_with_timezone(self):
+        # GH 15426
+        ts = Timestamp("2016-01-01 12:00:00", tz='US/Pacific')
+        exp = DatetimeIndex([ts, ts])
+        res = _possibly_downcast_to_dtype(exp, exp.dtype)
+        tm.assert_index_equal(res, exp)
+
+        res = _possibly_downcast_to_dtype(exp.asi8, exp.dtype)
+        tm.assert_index_equal(res, exp)
+
 
 class TestInferDtype(tm.TestCase):
 

pandas/types/cast.py

@@ -133,7 +133,8 @@ def trans(x):  # noqa
                 if dtype.tz:
                     # convert to datetime and change timezone
                     from pandas import to_datetime
-                    result = to_datetime(result).tz_localize(dtype.tz)
+                    result = to_datetime(result).tz_localize('utc')
+                    result = result.tz_convert(dtype.tz)
 
     except:
         pass