BUG: Do not round DatetimeIndex nanosecond precision when iterating #19628
Conversation
Codecov Report
@@ Coverage Diff @@
## master #19628 +/- ##
=======================================
Coverage 91.58% 91.58%
=======================================
Files 150 150
Lines 48806 48806
=======================================
Hits 44701 44701
Misses 4105 4105
Continue to review full report at Codecov.
|
pandas/_libs/tslibs/timezones.pyx
Outdated
| @@ -30,6 +32,10 @@ cdef int64_t NPY_NAT = get_nat() | |||
| # ---------------------------------------------------------------------- | |||
|
|
|||
| cdef inline bint is_utc(object tz): | |||
| if PY3: | |||
There was a problem hiding this comment.
this would be a perf issue, you can almost certainly access this in stead at the cython level
| from pandas import (DatetimeIndex, Index, date_range, DataFrame, | ||
| Timestamp, offsets) | ||
|
|
||
| from pandas.util.testing import assert_almost_equal | ||
|
|
||
| if PY3: | ||
| from datetime import timezone |
There was a problem hiding this comment.
this is not idiomatic, see comments above
There was a problem hiding this comment.
rather than put this here, make a fixture in pandas/conftest.py
called datetime_tz_utc to return this and skip if its not PY3
mroeschke
force-pushed
the
datetimeindex_iter
branch
from
700f83c to
1df622b
Compare
mroeschke
changed the title
|
I updated the fix and found the real issue (edited in my description above). |
| if box: | ||
| dt = Timestamp(dt) | ||
| result[i] = dt | ||
| # Python datetime objects do not support nanosecond |
There was a problem hiding this comment.
use tz_convert_tzlocal_to_utc here from conversion.pyx
| from pandas import (DatetimeIndex, Index, date_range, DataFrame, | ||
| Timestamp, offsets) | ||
|
|
||
| from pandas.util.testing import assert_almost_equal | ||
|
|
||
| if PY3: | ||
| from datetime import timezone |
There was a problem hiding this comment.
rather than put this here, make a fixture in pandas/conftest.py
called datetime_tz_utc to return this and skip if its not PY3
| @@ -258,6 +264,13 @@ def test_iteration_preserves_tz(self): | |||
| assert result._repr_base == expected._repr_base | |||
| assert result == expected | |||
|
|
|||
| @pytest.mark.parametrize('tz', [None, 'UTC', "US/Central", dt_UTC, | |||
| dateutil.tz.tzoffset(None, -28800)]) | |||
| def test_iteration_preserves_nanoseconds(self, tz): | |||
There was a problem hiding this comment.
add a similar test for pandas/tests/scalar/test_timezones.py
| @@ -258,6 +264,13 @@ def test_iteration_preserves_tz(self): | |||
| assert result._repr_base == expected._repr_base | |||
| assert result == expected | |||
|
|
|||
| @pytest.mark.parametrize('tz', [None, 'UTC', "US/Central", dt_UTC, | |||
| dateutil.tz.tzoffset(None, -28800)]) | |||
There was a problem hiding this comment.
move to test_timezones (in same subdir)
| def test_iteration_preserves_nanoseconds(self, tz): | ||
| # GH 19603 | ||
| index = pd.DatetimeIndex(["2018-02-08 15:00:00.168456358"], tz=tz) | ||
| assert list(index)[0] == index[0] |
There was a problem hiding this comment.
check both getitem and iteration
mroeschke
force-pushed
the
datetimeindex_iter
branch
from
1df622b to
61cc22b
Compare
|
Replaced fix with the use of |
Fix the issue within fix_offsets add addtional dateutil test Address_edits
mroeschke
force-pushed
the
datetimeindex_iter
branch
from
61cc22b to
14b8120
Compare
|
All green |
|
thanks @mroeschke FYI looking to move tz to be fixtures (e.g. the strings 'UTC', 'Asia/Tokyo') etc....across the test codebase, IOW to have a 'tz' fixture (prob need several slightly different ones) |
git diff upstream/master -u -- "*.py" | flake8 --diffThis issue was actually very specific to localizing aDatetimeIndexwithdatetime.timezone.utcI realize our timezone support relies heavily on pytz and dateutil, but I am curious how much support we have fordatetime.timezoneobjects.So the actual issue was that
datetime.timezone.utcis (rightly) considered a fixed offset, and the code path was calculating the new value with Pythondatetimesandtimedeltaswhich doesn't yet support nanosecond resolution.