API/ENH: tz_localize handling of nonexistent times: rename keyword + add shift option

doc/source/timeseries.rst

@@ -2305,6 +2305,31 @@ constructor as well as ``tz_localize``.
    # tz_convert(None) is identical with tz_convert('UTC').tz_localize(None)
    didx.tz_convert('UCT').tz_localize(None)
 
+.. _timeseries.timezone_nonexsistent:
+
+Nonexistent Times when Localizing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A DST transition may also shift the local time ahead by 1 hour creating nonexistent
+local times. The behavior of localizing a timeseries with nonexistent times
+can be controlled by the ``nonexistent`` argument. The following options are available:
+
+* ``shift``: Shifts nonexistent times forward to the closest real time
+* ``NaT``: Replaces nonexistent times with ``NaT``
+* ``raise``: Raises a ``pytz.NonExistentTimeError`` (the default behavior)
+
+.. ipython:: python
+    # 2:30 is a nonexistent time
+    dti = date_range(start='2015-03-29 01:30:00', periods=3, freq='H')
+    dti
+    dti.tz_localize('Europe/Warsaw', nonexistent='shift')
+    dti.tz_localize('Europe/Warsaw', nonexistent='NaT')
+
+.. code-block:: ipython
+
+   In [2]: dti.tz_localize('Europe/Warsaw')
+   NonExistentTimeError: 2015-03-29 02:30:00
+
 .. _timeseries.timezone_series:
 
 TZ Aware Dtypes

doc/source/whatsnew/v0.24.0.txt

@@ -185,6 +185,7 @@ Other Enhancements
 - :class:`Resampler` now is iterable like :class:`GroupBy` (:issue:`15314`).
 - :meth:`Series.resample` and :meth:`DataFrame.resample` have gained the :meth:`Resampler.quantile` (:issue:`15023`).
 - :meth:`Index.to_frame` now supports overriding column name(s) (:issue:`22580`).
+- :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` have gained the ``nonexistent`` argument for alternative handling of nonexistent times (:issue:`8917`)
 
 .. _whatsnew_0240.api_breaking:
 

pandas/_libs/tslibs/conversion.pyx

@@ -44,6 +44,7 @@ from nattype cimport NPY_NAT, checknull_with_nat
 # Constants
 
 cdef int64_t DAY_NS = 86400000000000LL
+cdef int64_t HOURS_NS = 3600000000000
 NS_DTYPE = np.dtype('M8[ns]')
 TD_DTYPE = np.dtype('m8[ns]')
 
@@ -826,7 +827,7 @@ def tz_convert(int64_t[:] vals, object tz1, object tz2):
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
-                       object errors='raise'):
+                       object nonexistent=None, object errors='raise'):
     """
     Localize tzinfo-naive i8 to given time zone (using pytz). If
     there are ambiguities in the values, raise AmbiguousTimeError.
@@ -837,6 +838,8 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
     tz : tzinfo or None
     ambiguous : str, bool, or arraylike
         If arraylike, must have the same length as vals
+    nonexistent : str, bool, or arraylike
+        If arraylike, must have the same length as vals
     errors : {"raise", "coerce"}, default "raise"
 
     Returns
@@ -849,10 +852,11 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
         ndarray ambiguous_array
         Py_ssize_t i, idx, pos, ntrans, n = len(vals)
         int64_t *tdata
-        int64_t v, left, right
+        int64_t v, left, right, val, v_left, v_right
         ndarray[int64_t] result, result_a, result_b, dst_hours
         npy_datetimestruct dts
         bint infer_dst = False, is_dst = False, fill = False
+        bint shift = False, fill_nonexist = False
         bint is_coerce = errors == 'coerce', is_raise = errors == 'raise'
 
     # Vectorized version of DstTzInfo.localize
@@ -888,39 +892,43 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
                              "the same size as vals")
         ambiguous_array = np.asarray(ambiguous)
 
+    if is_string_object(nonexistent):
+        if nonexistent == 'NaT':
+            fill_nonexist = True
+        elif nonexistent == 'shift':
+            shift = True
+
     trans, deltas, typ = get_dst_info(tz)
 
     tdata = <int64_t*> cnp.PyArray_DATA(trans)
     ntrans = len(trans)
 
+    # Determine whether each date lies left of the DST transition (store in
+    # result_a) or right of the DST transition (store in result_b)
     result_a = np.empty(n, dtype=np.int64)
     result_b = np.empty(n, dtype=np.int64)
     result_a.fill(NPY_NAT)
     result_b.fill(NPY_NAT)
 
-    # left side
-    idx_shifted = (np.maximum(0, trans.searchsorted(
+    idx_shifted_left = (np.maximum(0, trans.searchsorted(
         vals - DAY_NS, side='right') - 1)).astype(np.int64)
 
-    for i in range(n):
-        v = vals[i] - deltas[idx_shifted[i]]
-        pos = bisect_right_i8(tdata, v, ntrans) - 1
-
-        # timestamp falls to the left side of the DST transition
-        if v + deltas[pos] == vals[i]:
-            result_a[i] = v
-
-    # right side
-    idx_shifted = (np.maximum(0, trans.searchsorted(
+    idx_shifted_right = (np.maximum(0, trans.searchsorted(
         vals + DAY_NS, side='right') - 1)).astype(np.int64)
 
     for i in range(n):
-        v = vals[i] - deltas[idx_shifted[i]]
-        pos = bisect_right_i8(tdata, v, ntrans) - 1
+        val = vals[i]
+        v_left = val - deltas[idx_shifted_left[i]]
+        pos_left = bisect_right_i8(tdata, v_left, ntrans) - 1
+        # timestamp falls to the left side of the DST transition
+        if v_left + deltas[pos_left] == val:
+            result_a[i] = v_left
 
+        v_right = val - deltas[idx_shifted_right[i]]
+        pos_right = bisect_right_i8(tdata, v_right, ntrans) - 1
         # timestamp falls to the right side of the DST transition
-        if v + deltas[pos] == vals[i]:
-            result_b[i] = v
+        if v_right + deltas[pos_right] == val:
+            result_b[i] = v_right
 
     if infer_dst:
         dst_hours = np.empty(n, dtype=np.int64)
@@ -935,7 +943,7 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
             stamp = _render_tstamp(vals[trans_idx])
             raise pytz.AmbiguousTimeError(
                 "Cannot infer dst time from %s as there "
-                "are no repeated times" % stamp)
+                "are no repeated times".format(stamp))
         # Split the array into contiguous chunks (where the difference between
         # indices is 1).  These are effectively dst transitions in different
         # years which is useful for checking that there is not an ambiguous
@@ -960,18 +968,19 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
                 if switch_idx.size > 1:
                     raise pytz.AmbiguousTimeError(
                         "There are %i dst switches when "
-                        "there should only be 1." % switch_idx.size)
+                        "there should only be 1.".format(switch_idx.size))
                 switch_idx = switch_idx[0] + 1
                 # Pull the only index and adjust
                 a_idx = grp[:switch_idx]
                 b_idx = grp[switch_idx:]
                 dst_hours[grp] = np.hstack((result_a[a_idx], result_b[b_idx]))
 
     for i in range(n):
+        val = vals[i]
         left = result_a[i]
         right = result_b[i]
-        if vals[i] == NPY_NAT:
-            result[i] = vals[i]
+        if val == NPY_NAT:
+            result[i] = val
         elif left != NPY_NAT and right != NPY_NAT:
             if left == right:
                 result[i] = left
@@ -986,19 +995,27 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
                 elif fill:
                     result[i] = NPY_NAT
                 else:
-                    stamp = _render_tstamp(vals[i])
+                    stamp = _render_tstamp(val)
                     raise pytz.AmbiguousTimeError(
                         "Cannot infer dst time from %r, try using the "
-                        "'ambiguous' argument" % stamp)
+                        "'ambiguous' argument".format(stamp))
         elif left != NPY_NAT:
             result[i] = left
         elif right != NPY_NAT:
             result[i] = right
         else:
-            if is_coerce:
+            # Handle nonexistent times
+            if shift:
+                # Shift the nonexistent time forward to the closest existing
+                # time
+                remaining_minutes = val % HOURS_NS
+                new_local = val + (HOURS_NS - remaining_minutes)
+                delta_idx = trans.searchsorted(new_local, side='right') - 1
+                result[i] = new_local - deltas[delta_idx]
+            elif fill_nonexist or is_coerce:
                 result[i] = NPY_NAT
             else:
-                stamp = _render_tstamp(vals[i])
+                stamp = _render_tstamp(val)
                 raise pytz.NonExistentTimeError(stamp)
 
     return result

pandas/_libs/tslibs/nattype.pyx

@@ -535,6 +535,13 @@ class NaTType(_NaT):
             - 'NaT' will return NaT for an ambiguous time
             - 'raise' will raise an AmbiguousTimeError for an ambiguous time
 
+        nonexistent : 'shift', 'NaT', default 'raise'
+            - 'shift' will shift the nonexistent time forward to the closest
+              existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
+
         errors : 'raise', 'coerce', default 'raise'
             - 'raise' will raise a NonExistentTimeError if a timestamp is not
                valid in the specified timezone (e.g. due to a transition from

pandas/_libs/tslibs/timestamps.pyx

@@ -867,7 +867,8 @@ class Timestamp(_Timestamp):
     def is_leap_year(self):
         return bool(ccalendar.is_leapyear(self.year))
 
-    def tz_localize(self, tz, ambiguous='raise', errors='raise'):
+    def tz_localize(self, tz, ambiguous='raise', nonexistent='raise',
+                    errors='raise'):
         """
         Convert naive Timestamp to local time zone, or remove
         timezone from tz-aware Timestamp.
@@ -884,6 +885,13 @@ class Timestamp(_Timestamp):
             - 'NaT' will return NaT for an ambiguous time
             - 'raise' will raise an AmbiguousTimeError for an ambiguous time
 
+        nonexistent : 'shift', 'NaT', default 'raise'
+            - 'shift' will shift the nonexistent time forward to the closest
+              existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
+
         errors : 'raise', 'coerce', default 'raise'
             - 'raise' will raise a NonExistentTimeError if a timestamp is not
                valid in the specified timezone (e.g. due to a transition from
@@ -911,7 +919,9 @@ class Timestamp(_Timestamp):
             if not is_string_object(ambiguous):
                 ambiguous = [ambiguous]
             value = tz_localize_to_utc(np.array([self.value], dtype='i8'), tz,
-                                       ambiguous=ambiguous, errors=errors)[0]
+                                       ambiguous=ambiguous,
+                                       nonexistent=nonexistent,
+                                       errors=errors)[0]
             return Timestamp(value, tz=tz)
         else:
             if tz is None:

pandas/core/arrays/datetimes.py

@@ -616,7 +616,8 @@ def tz_convert(self, tz):
         # No conversion since timestamps are all UTC to begin with
         return self._shallow_copy(tz=tz)
 
-    def tz_localize(self, tz, ambiguous='raise', errors='raise'):
+    def tz_localize(self, tz, ambiguous='raise', nonexistent='raise',
+                    errors='raise'):
         """
         Localize tz-naive Datetime Array/Index to tz-aware
         Datetime Array/Index.
@@ -632,8 +633,7 @@ def tz_localize(self, tz, ambiguous='raise', errors='raise'):
         tz : string, pytz.timezone, dateutil.tz.tzfile or None
             Time zone to convert timestamps to. Passing ``None`` will
             remove the time zone information preserving local time.
-        ambiguous : str {'infer', 'NaT', 'raise'} or bool array,
-            default 'raise'
+        ambiguous : 'infer', 'NaT', bool array, default 'raise'
 
             - 'infer' will attempt to infer fall dst-transition hours based on
               order
@@ -644,6 +644,13 @@ def tz_localize(self, tz, ambiguous='raise', errors='raise'):
             - 'raise' will raise an AmbiguousTimeError if there are ambiguous
               times
 
+        nonexistent : 'shift', 'NaT' default 'raise'
+            - 'shift' will shift the nonexistent times forward to the closest
+              existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
+
         errors : {'raise', 'coerce'}, default 'raise'
 
             - 'raise' will raise a NonExistentTimeError if a timestamp is not
@@ -703,9 +710,10 @@ def tz_localize(self, tz, ambiguous='raise', errors='raise'):
             tz = timezones.maybe_get_tz(tz)
             # Convert to UTC
 
-            new_dates = conversion.tz_localize_to_utc(self.asi8, tz,
-                                                      ambiguous=ambiguous,
-                                                      errors=errors)
+            new_dates = conversion.tz_localize_to_utc(
+                self.asi8, tz, ambiguous=ambiguous, nonexistent=nonexistent,
+                errors=errors
+            )
         new_dates = new_dates.view(_NS_DTYPE)
         return self._shallow_copy(new_dates, tz=tz)
 

pandas/core/generic.py

@@ -8592,7 +8592,7 @@ def _tz_convert(ax, tz):
         return result.__finalize__(self)
 
     def tz_localize(self, tz, axis=0, level=None, copy=True,
-                    ambiguous='raise'):
+                    ambiguous='raise', nonexistent='raise'):
         """
         Localize tz-naive TimeSeries to target time zone.
 
@@ -8614,6 +8614,12 @@ def tz_localize(self, tz, axis=0, level=None, copy=True,
             - 'NaT' will return NaT where there are ambiguous times
             - 'raise' will raise an AmbiguousTimeError if there are ambiguous
               times
+        nonexistent : 'shift', 'NaT', default 'raise'
+            - 'shift' will shift the nonexistent times forward to the closest
+              existing time
+            - 'NaT' will return NaT where there are nonexistent times
+            - 'raise' will raise an NonExistentTimeError if there are
+              nonexistent times
 
         Returns
         -------
@@ -8626,7 +8632,7 @@ def tz_localize(self, tz, axis=0, level=None, copy=True,
         axis = self._get_axis_number(axis)
         ax = self._get_axis(axis)
 
-        def _tz_localize(ax, tz, ambiguous):
+        def _tz_localize(ax, tz, ambiguous, nonexistent):
             if not hasattr(ax, 'tz_localize'):
                 if len(ax) > 0:
                     ax_name = self._get_axis_name(axis)
@@ -8635,19 +8641,23 @@ def _tz_localize(ax, tz, ambiguous):
                 else:
                     ax = DatetimeIndex([], tz=tz)
             else:
-                ax = ax.tz_localize(tz, ambiguous=ambiguous)
+                ax = ax.tz_localize(
+                    tz, ambiguous=ambiguous, nonexistent=nonexistent
+                )
             return ax
 
         # if a level is given it must be a MultiIndex level or
         # equivalent to the axis name
         if isinstance(ax, MultiIndex):
             level = ax._get_level_number(level)
-            new_level = _tz_localize(ax.levels[level], tz, ambiguous)
+            new_level = _tz_localize(
+                ax.levels[level], tz, ambiguous, nonexistent
+            )
             ax = ax.set_levels(new_level, level=level)
         else:
             if level not in (None, 0, ax.name):
                 raise ValueError("The level {0} is not valid".format(level))
-            ax = _tz_localize(ax, tz, ambiguous)
+            ax = _tz_localize(ax, tz, ambiguous, nonexistent)
 
         result = self._constructor(self._data, copy=copy)
         result.set_axis(ax, axis=axis, inplace=True)

pandas/tests/indexes/datetimes/test_timezones.py

@@ -574,6 +574,24 @@ def test_dti_tz_localize_bdate_range(self):
         localized = dr.tz_localize(pytz.utc)
         tm.assert_index_equal(dr_utc, localized)
 
+    @pytest.mark.parametrize('tz', ['Europe/Warsaw', 'dateutil/Europe/Warsaw'])
+    @pytest.mark.parametrize('method, exp', [
+        ['shift', '2015-03-29 03:00:00'],
+        ['NaT', pd.NaT],
+        ['raise', None]
+    ])
+    def test_dti_tz_localize_nonexistent(self, tz, method, exp):
+        # GH 8917
+        n = 60
+        dti = date_range(start='2015-03-29 02:00:00', periods=n, freq='min')
+        if method == 'raise':
+            with pytest.raises(pytz.NonExistentTimeError):
+                dti.tz_localize(tz, nonexistent=method)
+        else:
+            result = dti.tz_localize(tz, nonexistent=method)
+            expected = DatetimeIndex([exp] * n, tz=tz)
+            tm.assert_index_equal(result, expected)
+
     # -------------------------------------------------------------
     # DatetimeIndex.normalize