BUG: Construct Timestamp with tz correctly near DST border

doc/source/whatsnew/v0.23.2.txt

@@ -73,6 +73,10 @@ Bug Fixes
 
 -
 
+**Timezones**
+- Bug in :class:`Timestamp` and :class:`DatetimeIndex` where passing a :class:`Timestamp` localized after a DST transition would return a datetime before the DST transition (:issue:`20854`)
+- Bug in comparing :class:`DataFrame`s with tz-aware :class:`DatetimeIndex` columns with a DST transition that raised a ``KeyError`` (:issue:`19970`)
+
 **Other**
 
 -

pandas/_libs/tslibs/conversion.pyx

@@ -347,25 +347,11 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz,
     if tz is not None:
         tz = maybe_get_tz(tz)
 
-        # sort of a temporary hack
         if ts.tzinfo is not None:
-            if hasattr(tz, 'normalize') and hasattr(ts.tzinfo, '_utcoffset'):
-                ts = tz.normalize(ts)
-                obj.value = pydatetime_to_dt64(ts, &obj.dts)
-                obj.tzinfo = ts.tzinfo
-            else:
-                # tzoffset
-                try:
-                    tz = ts.astimezone(tz).tzinfo
-                except:
-                    pass
-                obj.value = pydatetime_to_dt64(ts, &obj.dts)
-                ts_offset = get_utcoffset(ts.tzinfo, ts)
-                obj.value -= int(ts_offset.total_seconds() * 1e9)
-                tz_offset = get_utcoffset(tz, ts)
-                obj.value += int(tz_offset.total_seconds() * 1e9)
-                dt64_to_dtstruct(obj.value, &obj.dts)
-                obj.tzinfo = tz
+            # Convert the current timezone to the passed timezone
+            ts = ts.astimezone(tz)
+            obj.value = pydatetime_to_dt64(ts, &obj.dts)
+            obj.tzinfo = ts.tzinfo
         elif not is_utc(tz):
             ts = _localize_pydatetime(ts, tz)
             obj.value = pydatetime_to_dt64(ts, &obj.dts)

pandas/tests/frame/test_timezones.py

@@ -133,3 +133,13 @@ def test_frame_reset_index(self, tz):
         xp = df.index.tz
         rs = roundtripped.index.tz
         assert xp == rs
+
+    @pytest.mark.parametrize('tz', [None, 'America/New_York'])
+    def test_boolean_compare_transpose_tzindex_with_dst(self, tz):
+        # GH 19970
+        idx = date_range('20161101', '20161130', freq='4H', tz=tz)
+        df = DataFrame({'a': range(len(idx)), 'b': range(len(idx))},
+                       index=idx)
+        result = df.T == df.T
+        expected = DataFrame(True, index=list('ab'), columns=idx)
+        tm.assert_frame_equal(result, expected)

pandas/tests/indexes/datetimes/test_construction.py

@@ -469,6 +469,15 @@ def test_constructor_with_non_normalized_pytz(self, tz):
         result = DatetimeIndex(['2010'], tz=non_norm_tz)
         assert pytz.timezone(tz) is result.tz
 
+    def test_constructor_timestamp_near_dst(self):
+        # GH 20854
+        ts = [Timestamp('2016-10-30 03:00:00+0300', tz='Europe/Helsinki'),
+              Timestamp('2016-10-30 03:00:00+0200', tz='Europe/Helsinki')]
+        result = DatetimeIndex(ts)
+        expected = DatetimeIndex([ts[0].to_pydatetime(),
+                                  ts[1].to_pydatetime()])
+        tm.assert_index_equal(result, expected)
+
 
 class TestTimeSeries(object):
 

pandas/tests/indexes/datetimes/test_date_range.py

@@ -278,6 +278,20 @@ def test_wom_len(self, periods):
         res = date_range(start='20110101', periods=periods, freq='WOM-1MON')
         assert len(res) == periods
 
+    def test_construct_over_dst(self):
+        # GH 20854
+        pre_dst = Timestamp('2010-11-07 01:00:00').tz_localize('US/Pacific',
+                                                               ambiguous=True)
+        pst_dst = Timestamp('2010-11-07 01:00:00').tz_localize('US/Pacific',
+                                                               ambiguous=False)
+        expect_data = [Timestamp('2010-11-07 00:00:00', tz='US/Pacific'),
+                       pre_dst,
+                       pst_dst]
+        expected = DatetimeIndex(expect_data)
+        result = date_range(start='2010-11-7', periods=3,
+                            freq='H', tz='US/Pacific')
+        tm.assert_index_equal(result, expected)
+
 
 class TestGenRangeGeneration(object):
 

pandas/tests/scalar/timestamp/test_timestamp.py

@@ -528,6 +528,14 @@ def test_disallow_setting_tz(self, tz):
         with pytest.raises(AttributeError):
             ts.tz = tz
 
+    @pytest.mark.parametrize('offset', ['+0300', '+0200'])
+    def test_construct_timestamp_near_dst(self, offset):
+        # GH 20854
+        expected = Timestamp('2016-10-30 03:00:00{}'.format(offset),
+                             tz='Europe/Helsinki')
+        result = Timestamp(expected, tz='Europe/Helsinki')
+        assert result == expected
+
 
 class TestTimestamp(object):