BUG: to_datetime not localizing Series when utc=True (pandas-dev#6415)

Modify test case

Comment about test edit, move conversion logic to convert_listlike

Add new section in whatsnew and update test

Alter SQL tests

Modify whatsnew and make new wrapper function to handle UTC conversion

Simiplified whatsnew and reverted arg renaming

Author: mroeschke

doc/source/whatsnew/v0.21.0.txt

@@ -203,6 +203,39 @@ named ``.isna()`` and ``.notna()``, these are included for classes ``Categorical
 
 The configuration option ``pd.options.mode.use_inf_as_null`` is deprecated, and ``pd.options.mode.use_inf_as_na`` is added as a replacement.
 
+.. _whatsnew_0210.api.utc_localization_with_series:
+
+UTC Localization with Series
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  Previous Behavior
+
+  .. ipython:: python
+
+     s = Series(['20130101 00:00:00'] * 10)
+
+  .. code-block:: python
+
+     In [12]: pd.to_datetime(s, utc=True)
+     Out[12]:
+     0   2013-01-01
+     1   2013-01-01
+     2   2013-01-01
+     3   2013-01-01
+     4   2013-01-01
+     5   2013-01-01
+     6   2013-01-01
+     7   2013-01-01
+     8   2013-01-01
+     9   2013-01-01
+     dtype: datetime64[ns]
+
+  New Behavior
+
+  .. ipython:: python
+
+     pd.to_datetime(s, utc=True)
+
 .. _whatsnew_0210.api:
 
 Other API Changes
@@ -266,7 +299,6 @@ Conversion
 - Bug in assignment against datetime-like data with ``int`` may incorrectly converte to datetime-like (:issue:`14145`)
 - Bug in assignment against ``int64`` data with ``np.ndarray`` with ``float64`` dtype may keep ``int64`` dtype (:issue:`14001`)
 
-
 Indexing
 ^^^^^^^^
 

pandas/core/tools/datetimes.py

@@ -340,6 +340,17 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
 
     tz = 'utc' if utc else None
 
+    def _maybe_convert_to_utc(arg, utc):
+        if utc:
+            if isinstance(arg, ABCSeries):
+                arg = arg.dt.tz_localize('UTC')
+            elif isinstance(arg, DatetimeIndex):
+                if arg.tz is None:
+                    arg = arg.tz_localize('UTC')
+                else:
+                    arg = arg.tz_convert('UTC')
+        return arg
+
     def _convert_listlike(arg, box, format, name=None, tz=tz):
 
         if isinstance(arg, (list, tuple)):
@@ -359,7 +370,7 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
                     return DatetimeIndex(arg, tz=tz, name=name)
                 except ValueError:
                     pass
-
+            arg = _maybe_convert_to_utc(arg, utc)
             return arg
 
         elif unit is not None:
@@ -378,12 +389,15 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
         elif getattr(arg, 'ndim', 1) > 1:
             raise TypeError('arg must be a string, datetime, list, tuple, '
                             '1-d array, or Series')
-
+        # _ensure_object converts Series to numpy array, need to reconvert
+        # upon return
+        arg_is_series = isinstance(arg, ABCSeries)
         arg = _ensure_object(arg)
         require_iso8601 = False
 
         if infer_datetime_format and format is None:
-            format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
+            format = _guess_datetime_format_for_array(arg,
+                                                      dayfirst=dayfirst)
 
         if format is not None:
             # There is a special fast-path for iso8601 formatted
@@ -410,7 +424,8 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
                 # fallback
                 if result is None:
                     try:
-                        result = tslib.array_strptime(arg, format, exact=exact,
+                        result = tslib.array_strptime(arg, format,
+                                                      exact=exact,
                                                       errors=errors)
                     except tslib.OutOfBoundsDatetime:
                         if errors == 'raise':
@@ -434,9 +449,11 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
                     yearfirst=yearfirst,
                     require_iso8601=require_iso8601
                 )
-
             if is_datetime64_dtype(result) and box:
                 result = DatetimeIndex(result, tz=tz, name=name)
+            # GH 6415
+            elif arg_is_series:
+                result = _maybe_convert_to_utc(Series(result, name=name), utc)
             return result
 
         except ValueError as e:
@@ -506,7 +523,7 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
         result = arg
     elif isinstance(arg, ABCSeries):
         from pandas import Series
-        values = _convert_listlike(arg._values, False, format)
+        values = _convert_listlike(arg, False, format, name=arg.name)
         result = Series(values, index=arg.index, name=arg.name)
     elif isinstance(arg, (ABCDataFrame, MutableMapping)):
         result = _assemble_from_unit_mappings(arg, errors=errors)

pandas/tests/indexes/datetimes/test_tools.py

@@ -270,6 +270,38 @@ def test_to_datetime_utc_is_true(self):
         expected = pd.DatetimeIndex(data=date_range)
         tm.assert_index_equal(result, expected)
 
+    def test_to_datetime_utc_true_with_series(self):
+        # GH 6415: UTC=True with Series
+        data = ['20100102 121314', '20100102 121315']
+        expected_data = [pd.Timestamp('2010-01-02 12:13:14', tz='utc'),
+                         pd.Timestamp('2010-01-02 12:13:15', tz='utc')]
+        result = pd.to_datetime(pd.Series(data),
+                                format='%Y%m%d %H%M%S',
+                                utc=True)
+        expected = pd.Series(expected_data)
+        tm.assert_series_equal(result, expected)
+        result = pd.to_datetime(pd.Index(data),
+                                format='%Y%m%d %H%M%S',
+                                utc=True)
+        expected = pd.DatetimeIndex(expected_data)
+        tm.assert_index_equal(result, expected)
+
+        # GH 15760 UTC=True with Series
+        ts = 1.5e18
+        result = pd.to_datetime(pd.Series([ts]), utc=True)
+        expected = pd.Series([pd.Timestamp(ts, tz='utc')])
+        tm.assert_series_equal(result, expected)
+
+        test_dates = ['2013-01-01 00:00:00-01:00'] * 10
+        expected_data = [pd.Timestamp('20130101 01:00:00', tz='utc')] * 10
+        expected = pd.Series(expected_data)
+        ser = Series(test_dates)
+        result = pd.to_datetime(ser, utc=True)
+        tm.assert_series_equal(result, expected)
+        ser_naive = Series(test_dates, dtype='datetime64[ns]')
+        result = pd.to_datetime(ser_naive, utc=True)
+        tm.assert_series_equal(result, expected)
+
     def test_to_datetime_tz_psycopg2(self):
 
         # xref 8260

pandas/tests/io/test_sql.py

@@ -606,14 +606,15 @@ def test_date_parsing(self):
         # No Parsing
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn)
         assert not issubclass(df.DateCol.dtype.type, np.datetime64)
-
+        # Now that GH 6415 is fixed, dates are automatically parsed to UTC
+        utc_dtype = pd.core.dtypes.dtypes.DatetimeTZDtypeType
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 parse_dates=['DateCol'])
-        assert issubclass(df.DateCol.dtype.type, np.datetime64)
+        assert issubclass(df.DateCol.dtype.type, utc_dtype)
 
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 parse_dates={'DateCol': '%Y-%m-%d %H:%M:%S'})
-        assert issubclass(df.DateCol.dtype.type, np.datetime64)
+        assert issubclass(df.DateCol.dtype.type, utc_dtype)
 
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 parse_dates=['IntDateCol'])
@@ -631,8 +632,9 @@ def test_date_and_index(self):
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 index_col='DateCol',
                                 parse_dates=['DateCol', 'IntDateCol'])
-
-        assert issubclass(df.index.dtype.type, np.datetime64)
+        # Now that GH 6415 is fixed, dates are automatically parsed to UTC
+        utc_dtype = pd.core.dtypes.dtypes.DatetimeTZDtypeType
+        assert issubclass(df.index.dtype.type, utc_dtype)
         assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
 
     def test_timedelta(self):
@@ -1319,18 +1321,19 @@ def check(col):
     def test_date_parsing(self):
         # No Parsing
         df = sql.read_sql_table("types_test_data", self.conn)
-
+        # Now that GH 6415 is fixed, dates are automatically parsed to UTC
+        utc_dtype = pd.core.dtypes.dtypes.DatetimeTZDtypeType
         df = sql.read_sql_table("types_test_data", self.conn,
                                 parse_dates=['DateCol'])
-        assert issubclass(df.DateCol.dtype.type, np.datetime64)
+        assert issubclass(df.DateCol.dtype.type, utc_dtype)
 
         df = sql.read_sql_table("types_test_data", self.conn,
                                 parse_dates={'DateCol': '%Y-%m-%d %H:%M:%S'})
-        assert issubclass(df.DateCol.dtype.type, np.datetime64)
+        assert issubclass(df.DateCol.dtype.type, utc_dtype)
 
         df = sql.read_sql_table("types_test_data", self.conn, parse_dates={
             'DateCol': {'format': '%Y-%m-%d %H:%M:%S'}})
-        assert issubclass(df.DateCol.dtype.type, np.datetime64)
+        assert issubclass(df.DateCol.dtype.type, utc_dtype)
 
         df = sql.read_sql_table(
             "types_test_data", self.conn, parse_dates=['IntDateCol'])

pandas/tests/test_multilevel.py

@@ -2130,7 +2130,8 @@ def test_set_index_datetime(self):
                           '2011-07-19 08:00:00', '2011-07-19 09:00:00'],
              'value': range(6)})
         df.index = pd.to_datetime(df.pop('datetime'), utc=True)
-        df.index = df.index.tz_localize('UTC').tz_convert('US/Pacific')
+        # Removed 'tz_localize('utc') below after GH 6415 was fixed
+        df.index = df.index.tz_convert('US/Pacific')
 
         expected = pd.DatetimeIndex(['2011-07-19 07:00:00',
                                      '2011-07-19 08:00:00',