ENH: Parse %z and %Z directive in format for to_datetime (pandas-dev#19979)

Author: mroeschke

doc/source/whatsnew/v0.24.0.txt

@@ -12,7 +12,7 @@ New features
 
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
--
+- :func:`to_datetime` now supports the ``%Z`` and ``%z`` directive when passed into ``format`` (:issue:`13486`)
 -
 -
 

pandas/_libs/tslibs/strptime.pyx

@@ -20,6 +20,7 @@ except:
         except:
             from _dummy_thread import allocate_lock as _thread_allocate_lock
 
+import pytz
 
 from cython cimport Py_ssize_t
 from cpython cimport PyFloat_Check
@@ -40,6 +41,27 @@ from util cimport is_string_object
 from nattype cimport checknull_with_nat, NPY_NAT
 from nattype import nat_strings
 
+cdef dict _parse_code_table = {'y': 0,
+                               'Y': 1,
+                               'm': 2,
+                               'B': 3,
+                               'b': 4,
+                               'd': 5,
+                               'H': 6,
+                               'I': 7,
+                               'M': 8,
+                               'S': 9,
+                               'f': 10,
+                               'A': 11,
+                               'a': 12,
+                               'w': 13,
+                               'j': 14,
+                               'U': 15,
+                               'W': 16,
+                               'Z': 17,
+                               'p': 18,  # an additional key, only with I
+                               'z': 19}
+
 
 def array_strptime(ndarray[object] values, object fmt,
                    bint exact=True, errors='raise'):
@@ -58,15 +80,15 @@ def array_strptime(ndarray[object] values, object fmt,
         Py_ssize_t i, n = len(values)
         pandas_datetimestruct dts
         ndarray[int64_t] iresult
-        int year, month, day, minute, hour, second, weekday, julian, tz
-        int week_of_year, week_of_year_start
+        ndarray[object] result_timezone
+        int year, month, day, minute, hour, second, weekday, julian
+        int week_of_year, week_of_year_start, parse_code, ordinal
         int64_t us, ns
-        object val, group_key, ampm, found
+        object val, group_key, ampm, found, timezone
         dict found_key
         bint is_raise = errors=='raise'
         bint is_ignore = errors=='ignore'
         bint is_coerce = errors=='coerce'
-        int ordinal
 
     assert is_raise or is_ignore or is_coerce
 
@@ -79,6 +101,8 @@ def array_strptime(ndarray[object] values, object fmt,
                     in fmt):
                 raise ValueError("Cannot use '%W' or '%U' without "
                                  "day and year")
+        elif '%Z' in fmt and '%z' in fmt:
+            raise ValueError("Cannot parse both %Z and %z")
 
     global _TimeRE_cache, _regex_cache
     with _cache_lock:
@@ -108,32 +132,10 @@ def array_strptime(ndarray[object] values, object fmt,
 
     result = np.empty(n, dtype='M8[ns]')
     iresult = result.view('i8')
+    result_timezone = np.empty(n, dtype='object')
 
     dts.us = dts.ps = dts.as = 0
 
-    cdef dict _parse_code_table = {
-        'y': 0,
-        'Y': 1,
-        'm': 2,
-        'B': 3,
-        'b': 4,
-        'd': 5,
-        'H': 6,
-        'I': 7,
-        'M': 8,
-        'S': 9,
-        'f': 10,
-        'A': 11,
-        'a': 12,
-        'w': 13,
-        'j': 14,
-        'U': 15,
-        'W': 16,
-        'Z': 17,
-        'p': 18   # just an additional key, works only with I
-    }
-    cdef int parse_code
-
     for i in range(n):
         val = values[i]
         if is_string_object(val):
@@ -176,7 +178,7 @@ def array_strptime(ndarray[object] values, object fmt,
         year = 1900
         month = day = 1
         hour = minute = second = ns = us = 0
-        tz = -1
+        timezone = None
         # Default to -1 to signify that values not known; not critical to have,
         # though
         week_of_year = -1
@@ -266,21 +268,10 @@ def array_strptime(ndarray[object] values, object fmt,
                     # W starts week on Monday.
                     week_of_year_start = 0
             elif parse_code == 17:
-                # Since -1 is default value only need to worry about setting tz
-                # if it can be something other than -1.
-                found_zone = found_dict['Z'].lower()
-                for value, tz_values in enumerate(locale_time.timezone):
-                    if found_zone in tz_values:
-                        # Deal w/ bad locale setup where timezone names are the
-                        # same and yet time.daylight is true; too ambiguous to
-                        # be able to tell what timezone has daylight savings
-                        if (time.tzname[0] == time.tzname[1] and
-                            time.daylight and found_zone not in (
-                                "utc", "gmt")):
-                            break
-                        else:
-                            tz = value
-                            break
+                timezone = pytz.timezone(found_dict['Z'])
+            elif parse_code == 19:
+                timezone = parse_timezone_directive(found_dict['z'])
+
         # If we know the wk of the year and what day of that wk, we can figure
         # out the Julian day of the year.
         if julian == -1 and week_of_year != -1 and weekday != -1:
@@ -330,7 +321,9 @@ def array_strptime(ndarray[object] values, object fmt,
                 continue
             raise
 
-    return result
+        result_timezone[i] = timezone
+
+    return result, result_timezone
 
 
 """_getlang, LocaleTime, TimeRE, _calc_julian_from_U_or_W are vendored
@@ -538,14 +531,13 @@ class TimeRE(dict):
             # XXX: Does 'Y' need to worry about having less or more than
             #     4 digits?
             'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)",
             'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
             'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
             'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
             'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
             'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
-            'Z': self.__seqToRE([tz for tz_names in self.locale_time.timezone
-                                 for tz in tz_names],
-                                'Z'),
+            'Z': self.__seqToRE(pytz.all_timezones, 'Z'),
             '%': '%'})
         base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
         base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
@@ -632,3 +624,50 @@ cdef _calc_julian_from_U_or_W(int year, int week_of_year,
     else:
         days_to_week = week_0_length + (7 * (week_of_year - 1))
         return 1 + days_to_week + day_of_week
+
+cdef parse_timezone_directive(object z):
+    """
+    Parse the '%z' directive and return a pytz.FixedOffset
+
+    Parameters
+    ----------
+    z : string of the UTC offset
+
+    Returns
+    -------
+    pytz.FixedOffset
+
+    Notes
+    -----
+    This is essentially similar to the cpython implementation
+    https://github.com/python/cpython/blob/master/Lib/_strptime.py#L457-L479
+    """
+
+    cdef:
+        int gmtoff_fraction, hours, minutes, seconds, pad_number, microseconds
+        int total_minutes
+        object gmtoff_remainder, gmtoff_remainder_padding
+
+    if z == 'Z':
+        return pytz.FixedOffset(0)
+    if z[3] == ':':
+        z = z[:3] + z[4:]
+        if len(z) > 5:
+            if z[5] != ':':
+                msg = "Inconsistent use of : in {0}"
+                raise ValueError(msg.format(z))
+            z = z[:5] + z[6:]
+    hours = int(z[1:3])
+    minutes = int(z[3:5])
+    seconds = int(z[5:7] or 0)
+
+    # Pad to always return microseconds.
+    gmtoff_remainder = z[8:]
+    pad_number = 6 - len(gmtoff_remainder)
+    gmtoff_remainder_padding = "0" * pad_number
+    microseconds = int(gmtoff_remainder + gmtoff_remainder_padding)
+
+    total_minutes = ((hours * 60) + minutes + (seconds / 60) +
+                     (microseconds / 60000000))
+    total_minutes = -total_minutes if z.startswith("-") else total_minutes
+    return pytz.FixedOffset(total_minutes)

pandas/core/tools/datetimes.py

@@ -1,7 +1,8 @@
 from datetime import datetime, timedelta, time
-import numpy as np
 from collections import MutableMapping
 
+import numpy as np
+
 from pandas._libs import tslib
 from pandas._libs.tslibs.strptime import array_strptime
 from pandas._libs.tslibs import parsing, conversion
@@ -27,6 +28,7 @@
     ABCDataFrame)
 from pandas.core.dtypes.missing import notna
 from pandas.core import algorithms
+from pandas.compat import zip
 
 
 def _guess_datetime_format_for_array(arr, **kwargs):
@@ -103,6 +105,41 @@ def _convert_and_box_cache(arg, cache_array, box, errors, name=None):
     return result.values
 
 
+def _return_parsed_timezone_results(result, timezones, box, tz):
+    """
+    Return results from array_strptime if a %z or %Z directive was passed.
+
+    Parameters
+    ----------
+    result : ndarray
+        int64 date representations of the dates
+    timezones : ndarray
+        pytz timezone objects
+    box : boolean
+        True boxes result as an Index-like, False returns an ndarray
+    tz : object
+        None or pytz timezone object
+    Returns
+    -------
+    tz_result : ndarray of parsed dates with timezone
+        Returns:
+
+        - Index-like if box=True
+        - ndarray of Timestamps if box=False
+
+    """
+    if tz is not None:
+        raise ValueError("Cannot pass a tz argument when "
+                         "parsing strings with timezone "
+                         "information.")
+    tz_results = np.array([tslib.Timestamp(res).tz_localize(zone) for res, zone
+                           in zip(result, timezones)])
+    if box:
+        from pandas import Index
+        return Index(tz_results)
+    return tz_results
+
+
 def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
                 utc=None, box=True, format=None, exact=True,
                 unit=None, infer_datetime_format=False, origin='unix',
@@ -343,8 +380,11 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
                 # fallback
                 if result is None:
                     try:
-                        result = array_strptime(arg, format, exact=exact,
-                                                errors=errors)
+                        result, timezones = array_strptime(
+                            arg, format, exact=exact, errors=errors)
+                        if '%Z' in format or '%z' in format:
+                            return _return_parsed_timezone_results(
+                                result, timezones, box, tz)
                     except tslib.OutOfBoundsDatetime:
                         if errors == 'raise':
                             raise

pandas/tests/indexes/datetimes/test_tools.py

@@ -179,6 +179,59 @@ def test_to_datetime_format_weeks(self, cache):
         for s, format, dt in data:
             assert to_datetime(s, format=format, cache=cache) == dt
 
+    @pytest.mark.parametrize("box,const,assert_equal", [
+        [True, pd.Index, 'assert_index_equal'],
+        [False, np.array, 'assert_numpy_array_equal']])
+    @pytest.mark.parametrize("fmt,dates,expected_dates", [
+        ['%Y-%m-%d %H:%M:%S %Z',
+         ['2010-01-01 12:00:00 UTC'] * 2,
+         [pd.Timestamp('2010-01-01 12:00:00', tz='UTC')] * 2],
+        ['%Y-%m-%d %H:%M:%S %Z',
+         ['2010-01-01 12:00:00 UTC',
+          '2010-01-01 12:00:00 GMT',
+          '2010-01-01 12:00:00 US/Pacific'],
+         [pd.Timestamp('2010-01-01 12:00:00', tz='UTC'),
+          pd.Timestamp('2010-01-01 12:00:00', tz='GMT'),
+          pd.Timestamp('2010-01-01 12:00:00', tz='US/Pacific')]],
+        ['%Y-%m-%d %H:%M:%S%z',
+         ['2010-01-01 12:00:00+0100'] * 2,
+         [pd.Timestamp('2010-01-01 12:00:00',
+                       tzinfo=pytz.FixedOffset(60))] * 2],
+        ['%Y-%m-%d %H:%M:%S %z',
+         ['2010-01-01 12:00:00 +0100'] * 2,
+         [pd.Timestamp('2010-01-01 12:00:00',
+                       tzinfo=pytz.FixedOffset(60))] * 2],
+        ['%Y-%m-%d %H:%M:%S %z',
+         ['2010-01-01 12:00:00 +0100', '2010-01-01 12:00:00 -0100'],
+         [pd.Timestamp('2010-01-01 12:00:00',
+                       tzinfo=pytz.FixedOffset(60)),
+          pd.Timestamp('2010-01-01 12:00:00',
+                       tzinfo=pytz.FixedOffset(-60))]],
+        ['%Y-%m-%d %H:%M:%S %z',
+         ['2010-01-01 12:00:00 Z', '2010-01-01 12:00:00 Z'],
+         [pd.Timestamp('2010-01-01 12:00:00',
+                       tzinfo=pytz.FixedOffset(0)),  # pytz coerces to UTC
+          pd.Timestamp('2010-01-01 12:00:00',
+                       tzinfo=pytz.FixedOffset(0))]]])
+    def test_to_datetime_parse_tzname_or_tzoffset(self, box, const,
+                                                  assert_equal, fmt,
+                                                  dates, expected_dates):
+        # GH 13486
+        result = pd.to_datetime(dates, format=fmt, box=box)
+        expected = const(expected_dates)
+        getattr(tm, assert_equal)(result, expected)
+
+        with pytest.raises(ValueError):
+            pd.to_datetime(dates, format=fmt, box=box, utc=True)
+
+    @pytest.mark.parametrize('offset', [
+        '+0', '-1foo', 'UTCbar', ':10', '+01:000:01', ''])
+    def test_to_datetime_parse_timezone_malformed(self, offset):
+        fmt = '%Y-%m-%d %H:%M:%S %z'
+        date = '2010-01-01 12:00:00 ' + offset
+        with pytest.raises(ValueError):
+            pd.to_datetime([date], format=fmt)
+
 
 class TestToDatetime(object):
     def test_to_datetime_pydatetime(self):