BUG: Do not round DatetimeIndex nanosecond precision when iterating (pandas-dev#19628)

Author: mroeschke

doc/source/whatsnew/v0.23.0.txt

@@ -729,6 +729,7 @@ Timezones
 - Bug in :func:`DatetimeIndex.insert` where inserting ``NaT`` into a timezone-aware index incorrectly raised (:issue:`16357`)
 - Bug in the :class:`DataFrame` constructor, where tz-aware Datetimeindex and a given column name will result in an empty ``DataFrame`` (:issue:`19157`)
 - Bug in :func:`Timestamp.tz_localize` where localizing a timestamp near the minimum or maximum valid values could overflow and return a timestamp with an incorrect nanosecond value (:issue:`12677`)
+- Bug when iterating over :class:`DatetimeIndex` that was localized with fixed timezone offset that rounded nanosecond precision to microseconds (:issue:`19603`)
 
 Offsets
 ^^^^^^^

pandas/_libs/tslib.pyx

@@ -46,7 +46,8 @@ from tslibs.timezones cimport (is_utc, is_tzlocal, is_fixed_offset,
                                treat_tz_as_pytz, get_dst_info)
 from tslibs.conversion cimport (tz_convert_single, _TSObject,
                                 convert_datetime_to_tsobject,
-                                get_datetime64_nanos)
+                                get_datetime64_nanos,
+                                tz_convert_utc_to_tzlocal)
 from tslibs.conversion import tz_convert_single
 
 from tslibs.nattype import NaT, nat_strings, iNaT
@@ -144,12 +145,12 @@ def ints_to_pydatetime(ndarray[int64_t] arr, tz=None, freq=None,
                 if value == NPY_NAT:
                     result[i] = NaT
                 else:
-                    dt64_to_dtstruct(value, &dts)
-                    dt = create_datetime_from_ts(value, dts, tz, freq)
-                    dt = dt + tz.utcoffset(dt)
-                    if box:
-                        dt = Timestamp(dt)
-                    result[i] = dt
+                    # Python datetime objects do not support nanosecond
+                    # resolution (yet, PEP 564). Need to compute new value
+                    # using the i8 representation.
+                    local_value = tz_convert_utc_to_tzlocal(value, tz)
+                    dt64_to_dtstruct(local_value, &dts)
+                    result[i] = func_create(value, dts, tz, freq)
         else:
             trans, deltas, typ = get_dst_info(tz)
 

pandas/conftest.py

@@ -93,3 +93,9 @@ def compression_no_zip(request):
     except zip
     """
     return request.param
+
+
+@pytest.fixture(scope='module')
+def datetime_tz_utc():
+    from datetime import timezone
+    return timezone.utc

pandas/tests/indexes/datetimes/test_timezones.py

@@ -17,7 +17,7 @@
 import pandas as pd
 from pandas._libs import tslib
 from pandas._libs.tslibs import timezones
-from pandas.compat import lrange, zip
+from pandas.compat import lrange, zip, PY3
 from pandas import (DatetimeIndex, date_range, bdate_range,
                     Timestamp, isna, to_datetime, Index)
 
@@ -949,6 +949,17 @@ def test_dti_union_aware(self):
         result = rng.union(rng2)
         assert result.tz.zone == 'UTC'
 
+    @pytest.mark.parametrize('tz', [None, 'UTC', "US/Central",
+                                    dateutil.tz.tzoffset(None, -28800)])
+    @pytest.mark.usefixtures("datetime_tz_utc")
+    @pytest.mark.skipif(not PY3, reason="datetime.timezone not in PY2")
+    def test_iteration_preserves_nanoseconds(self, tz):
+        # GH 19603
+        index = DatetimeIndex(["2018-02-08 15:00:00.168456358",
+                               "2018-02-08 15:00:00.168456359"], tz=tz)
+        for i, ts in enumerate(index):
+            assert ts == index[i]
+
 
 class TestDateRange(object):
     """Tests for date_range with timezones"""