ENH: handling of dateutil-generated UTC datetime objects in to_datetime close #1693

Author: wesm

RELEASE.rst

@@ -30,6 +30,7 @@ pandas 0.8.2
 **Improvements to existing features**
 
   - Add ``flags`` option for ``re.compile`` in some Series.str methods (#1659)
+  - Parsing of UTC date strings in read_* functions (#1693)
 
 **API Changes**
 

pandas/io/tests/test_parsers.py

@@ -1388,6 +1388,16 @@ def test_file(self):
         url_table = read_table('file://localhost/'+localtable)
         assert_frame_equal(url_table, local_table)
 
+    def test_parse_tz_aware(self):
+        import pytz
+        # #1693
+        data = StringIO("Date,x\n2012-06-13T01:39:00Z,0.5")
+
+        # it works
+        result = read_csv(data, index_col=0, parse_dates=True)
+        stamp = result.index[0]
+        self.assert_(stamp.minute == 39)
+        self.assert_(result.index.tz is pytz.utc)
 
 class TestParseSQL(unittest.TestCase):
 

pandas/src/datetime.pyx

@@ -38,7 +38,7 @@ def ints_to_pydatetime(ndarray[int64_t] arr, tz=None):
         ndarray[object] result = np.empty(n, dtype=object)
 
     if tz is not None:
-        if tz is pytz.utc:
+        if _is_utc(tz):
             for i in range(n):
                 pandas_datetime_to_datetimestruct(arr[i], PANDAS_FR_ns, &dts)
                 result[i] = datetime(dts.year, dts.month, dts.day, dts.hour,
@@ -529,7 +529,7 @@ cpdef convert_to_tsobject(object ts, object tz=None):
         else:
             obj.value = _pydatetime_to_dts(ts, &obj.dts)
             obj.tzinfo = ts.tzinfo
-            if obj.tzinfo is not None:
+            if obj.tzinfo is not None and not _is_utc(obj.tzinfo):
                 obj.value -= _delta_to_nanoseconds(obj.tzinfo._utcoffset)
         _check_dts_bounds(obj.value, &obj.dts)
         return obj
@@ -543,7 +543,7 @@ cpdef convert_to_tsobject(object ts, object tz=None):
         _check_dts_bounds(obj.value, &obj.dts)
 
     if tz is not None:
-        if tz is pytz.utc:
+        if _is_utc(tz):
             obj.tzinfo = tz
         else:
             # Adjust datetime64 timestamp, recompute datetimestruct
@@ -558,6 +558,9 @@ cpdef convert_to_tsobject(object ts, object tz=None):
 
     return obj
 
+cdef inline bint _is_utc(object tz):
+    return tz is UTC or isinstance(tz, _du_utc)
+
 cdef int64_t _NS_LOWER_BOUND = -9223285636854775809LL
 cdef int64_t _NS_UPPER_BOUND = -9223372036854775807LL
 
@@ -788,6 +791,7 @@ def i8_to_pydt(int64_t i8, object tzinfo = None):
 # time zone conversion helpers
 
 try:
+    from dateutil.tz import tzutc as _du_utc
     import pytz
     UTC = pytz.utc
     have_pytz = True
@@ -937,7 +941,7 @@ def tz_localize_check(ndarray[int64_t] vals, object tz):
     if not have_pytz:
         raise Exception("Could not find pytz module")
 
-    if tz == pytz.utc or tz is None:
+    if tz == UTC or tz is None:
         return
 
     trans = _get_transitions(tz)
@@ -984,7 +988,7 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz):
     if not have_pytz:
         raise Exception("Could not find pytz module")
 
-    if tz == pytz.utc or tz is None:
+    if tz == UTC or tz is None:
         return vals
 
     trans = _get_transitions(tz)  # transition dates

pandas/tseries/index.py

@@ -213,18 +213,27 @@ def __new__(cls, data=None,
             else:
                 subarr = data.view(_NS_DTYPE)
         else:
-            subarr = tools.to_datetime(data)
+            try:
+                subarr = tools.to_datetime(data)
+            except ValueError:
+                # tz aware
+                subarr = tools.to_datetime(data, utc=True)
+
             if not np.issubdtype(subarr.dtype, np.datetime64):
                 raise TypeError('Unable to convert %s to datetime dtype'
                                 % str(data))
 
-        if tz is not None:
-            tz = tools._maybe_get_tz(tz)
-            # Convert local to UTC
-            ints = subarr.view('i8')
+        if isinstance(subarr, DatetimeIndex):
+            if tz is None:
+                tz = subarr.tz
+        else:
+            if tz is not None:
+                tz = tools._maybe_get_tz(tz)
+                # Convert local to UTC
+                ints = subarr.view('i8')
 
-            subarr = lib.tz_localize_to_utc(ints, tz)
-            subarr = subarr.view(_NS_DTYPE)
+                subarr = lib.tz_localize_to_utc(ints, tz)
+                subarr = subarr.view(_NS_DTYPE)
 
         subarr = subarr.view(cls)
         subarr.name = name

pandas/tseries/tests/test_timezones.py

@@ -369,6 +369,13 @@ def test_convert_tz_aware_datetime_datetime(self):
         self.assert_(np.array_equal(converted.asi8, ex_vals))
         self.assert_(converted.tz is pytz.utc)
 
+    def test_to_datetime_utc(self):
+        from dateutil.parser import parse
+        arr = np.array([parse('2012-06-13T01:39:00Z')], dtype=object)
+
+        result = to_datetime(arr, utc=True)
+        self.assert_(result.tz is pytz.utc)
+
 class TestTimeZones(unittest.TestCase):
 
     def setUp(self):