BUG: Construct Timestamp with tz correctly near DST border

doc/source/whatsnew/v0.23.1.txt

@@ -129,6 +129,10 @@ Bug Fixes
 - Bug in :func:`concat` where error was raised in concatenating :class:`Series` with numpy scalar and tuple names (:issue:`21015`)
 - Bug in :func:`concat` warning message providing the wrong guidance for future behavior (:issue:`21101`)
 
+**Timezones**
+- Bug in :class:`Timestamp` and :class:`DatetimeIndex` where passing a :class:`Timestamp` localized after a DST transition would return a datetime before the DST transition (:issue:`20854`)
+- Bug in comparing :class:`DataFrame`s with tz-aware :class:`DatetimeIndex` columns with a DST transition that raised a ``KeyError`` (:issue:`19970`)
+
 **Other**
 
 - Tab completion on :class:`Index` in IPython no longer outputs deprecation warnings (:issue:`21125`)

pandas/_libs/tslibs/conversion.pyx

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # cython: profile=False
 
+from warnings import catch_warnings
+
 cimport cython
 from cython cimport Py_ssize_t
 
@@ -347,9 +349,14 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz,
     if tz is not None:
         tz = maybe_get_tz(tz)
 
-        # sort of a temporary hack
         if ts.tzinfo is not None:
             if hasattr(tz, 'normalize') and hasattr(ts.tzinfo, '_utcoffset'):
+                # tz.localize does not correctly localize Timestamps near DST
+                # but correctly localizes datetimes
+                if hasattr(ts, 'to_pydatetime'):
+                    nanos += ts.nanosecond
+                    with catch_warnings(record=True):
+                        ts = ts.to_pydatetime()
                 ts = tz.normalize(ts)
                 obj.value = pydatetime_to_dt64(ts, &obj.dts)
                 obj.tzinfo = ts.tzinfo

pandas/tests/frame/test_operators.py

@@ -992,6 +992,16 @@ def test_boolean_comparison(self):
         result = df == tup
         assert_frame_equal(result, expected)
 
+    @pytest.mark.parametrize('tz', [None, 'America/New_York'])
+    def test_boolean_compare_transpose_tzindex_with_dst(self, tz):
+        # GH 19970
+        idx = date_range('20161101', '20161130', freq='4H', tz=tz)
+        df = DataFrame({'a': range(len(idx)), 'b': range(len(idx))},
+                       index=idx)
+        result = df.T == df.T
+        expected = DataFrame(True, index=list('ab'), columns=idx)
+        assert_frame_equal(result, expected)
+
     def test_combine_generic(self):
         df1 = self.frame
         df2 = self.frame.loc[self.frame.index[:-5], ['A', 'B', 'C']]

pandas/tests/indexes/datetimes/test_construction.py

@@ -469,6 +469,15 @@ def test_constructor_with_non_normalized_pytz(self, tz):
         result = DatetimeIndex(['2010'], tz=non_norm_tz)
         assert pytz.timezone(tz) is result.tz
 
+    def test_constructor_timestamp_near_dst(self):
+        # GH 20854
+        ts = [Timestamp('2016-10-30 03:00:00+0300', tz='Europe/Helsinki'),
+              Timestamp('2016-10-30 03:00:00+0200', tz='Europe/Helsinki')]
+        result = DatetimeIndex(ts)
+        expected = DatetimeIndex([ts[0].to_pydatetime(),
+                                  ts[1].to_pydatetime()])
+        tm.assert_index_equal(result, expected)
+
 
 class TestTimeSeries(object):
 

pandas/tests/indexes/datetimes/test_date_range.py

@@ -278,6 +278,20 @@ def test_wom_len(self, periods):
         res = date_range(start='20110101', periods=periods, freq='WOM-1MON')
         assert len(res) == periods
 
+    def test_construct_over_dst(self):
+        # GH 20854
+        pre_dst = Timestamp('2010-11-07 01:00:00').tz_localize('US/Pacific',
+                                                               ambiguous=True)
+        pst_dst = Timestamp('2010-11-07 01:00:00').tz_localize('US/Pacific',
+                                                               ambiguous=False)
+        expect_data = [Timestamp('2010-11-07 00:00:00', tz='US/Pacific'),
+                       pre_dst,
+                       pst_dst]
+        expected = DatetimeIndex(expect_data)
+        result = date_range(start='2010-11-7', periods=3,
+                            freq='H', tz='US/Pacific')
+        tm.assert_index_equal(result, expected)
+
 
 class TestGenRangeGeneration(object):
 

pandas/tests/scalar/timestamp/test_timestamp.py

@@ -528,6 +528,14 @@ def test_disallow_setting_tz(self, tz):
         with pytest.raises(AttributeError):
             ts.tz = tz
 
+    @pytest.mark.parametrize('offset', ['+0300', '+0200'])
+    def test_construct_timestamp_near_dst(self, offset):
+        # GH 20854
+        expected = Timestamp('2016-10-30 03:00:00{}'.format(offset),
+                             tz='Europe/Helsinki')
+        result = Timestamp(expected, tz='Europe/Helsinki')
+        assert result == expected
+
 
 class TestTimestamp(object):