From 5cf8073bd7248bd3b8553478a8885c117ad0abc6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Nov 2018 18:24:52 -0800 Subject: [PATCH 01/19] Add more timezone fixtures --- pandas/_libs/tslibs/conversion.pyx | 8 ++++++-- pandas/_libs/tslibs/timezones.pyx | 2 +- pandas/conftest.py | 9 +++++---- pandas/tests/indexes/datetimes/test_timezones.py | 5 +---- pandas/tests/indexes/datetimes/test_tools.py | 8 ++++---- pandas/tests/scalar/timestamp/test_timezones.py | 6 ++---- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index c2897e1d0e8c8..5a33f91ffca5a 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -785,7 +785,8 @@ cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, Py_ssize_t i, n = len(vals) int64_t val - if get_timezone(tz) != 'UTC': + if not is_utc(get_timezone(tz)): + #if get_timezone(tz) != 'UTC': converted = np.empty(n, dtype=np.int64) if is_tzlocal(tz): for i in range(n): @@ -890,7 +891,10 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None, if is_tzlocal(tz): for i in range(n): v = vals[i] - result[i] = _tz_convert_tzlocal_utc(v, tz, to_utc=True) + if v == NPY_NAT: + result[i] = NPY_NAT + else: + result[i] = _tz_convert_tzlocal_utc(v, tz, to_utc=True) return result if is_string_object(ambiguous): diff --git a/pandas/_libs/tslibs/timezones.pyx b/pandas/_libs/tslibs/timezones.pyx index a2a40a8aa1ca4..5fa8a45af3083 100644 --- a/pandas/_libs/tslibs/timezones.pyx +++ b/pandas/_libs/tslibs/timezones.pyx @@ -58,7 +58,7 @@ cpdef inline object get_timezone(object tz): UJSON/pytables. maybe_get_tz (below) is the inverse of this process. """ if is_utc(tz): - return 'UTC' + return tz else: if treat_tz_as_dateutil(tz): if '.tar.gz' in tz._filename: diff --git a/pandas/conftest.py b/pandas/conftest.py index 479471332a274..50bf2cf342fa3 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -1,12 +1,12 @@ import importlib import os -from dateutil.tz import tzutc +from dateutil.tz import tzutc, tzlocal import hypothesis from hypothesis import strategies as st import numpy as np import pytest -from pytz import utc +from pytz import utc, FixedOffset from pandas.compat import PY3 import pandas.util._test_decorators as td @@ -245,7 +245,7 @@ def datetime_tz_utc(): return timezone.utc -utc_objs = ['utc', utc, tzutc()] +utc_objs = ['utc', 'dateutil/UTC', utc, tzutc()] if PY3: from datetime import timezone utc_objs.append(timezone.utc) @@ -354,7 +354,8 @@ def unique_nulls_fixture(request): TIMEZONES = [None, 'UTC', 'US/Eastern', 'Asia/Tokyo', 'dateutil/US/Pacific', - 'dateutil/Asia/Singapore'] + 'dateutil/Asia/Singapore', tzutc(), tzlocal(), FixedOffset(300), + FixedOffset(0)] @td.parametrize_fixture_doc(str(TIMEZONES)) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index 9ad540b174438..52f4f3c309038 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -774,10 +774,7 @@ def test_time_accessor(self, dtype): def test_timetz_accessor(self, tz_naive_fixture): # GH21358 - if tz_naive_fixture is not None: - tz = dateutil.tz.gettz(tz_naive_fixture) - else: - tz = None + tz = timezones.maybe_get_tz(tz_naive_fixture) expected = np.array([time(10, 20, 30, tzinfo=tz), pd.NaT]) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index c24c1025ea63c..e31d63ff8c3b9 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -409,8 +409,8 @@ def test_to_datetime_tz_pytz(self, cache): hour=3, minute=0))], dtype=object) result = pd.to_datetime(arr, utc=True, cache=cache) - expected = DatetimeIndex(['2000-01-01 08:00:00+00:00', - '2000-06-01 07:00:00+00:00'], + expected = DatetimeIndex(['2000-01-01 08:00:00', + '2000-06-01 07:00:00'], dtype='datetime64[ns, UTC]', freq=None) tm.assert_index_equal(result, expected) @@ -488,8 +488,8 @@ def test_to_datetime_tz_psycopg2(self, cache): dtype=object) result = pd.to_datetime(arr, errors='coerce', utc=True, cache=cache) - expected = DatetimeIndex(['2000-01-01 08:00:00+00:00', - '2000-06-01 07:00:00+00:00'], + expected = DatetimeIndex(['2000-01-01 08:00:00', + '2000-06-01 07:00:00'], dtype='datetime64[ns, UTC]', freq=None) tm.assert_index_equal(result, expected) diff --git a/pandas/tests/scalar/timestamp/test_timezones.py b/pandas/tests/scalar/timestamp/test_timezones.py index 6755d0bd4ae27..a859c081877b0 100644 --- a/pandas/tests/scalar/timestamp/test_timezones.py +++ b/pandas/tests/scalar/timestamp/test_timezones.py @@ -11,6 +11,7 @@ import dateutil from dateutil.tz import gettz, tzoffset +from pandas._libs.tslibs import timezones import pandas.util.testing as tm import pandas.util._test_decorators as td @@ -342,10 +343,7 @@ def test_timestamp_add_timedelta_push_over_dst_boundary(self, tz): def test_timestamp_timetz_equivalent_with_datetime_tz(self, tz_naive_fixture): # GH21358 - if tz_naive_fixture is not None: - tz = dateutil.tz.gettz(tz_naive_fixture) - else: - tz = None + tz = timezones.maybe_get_tz(tz_naive_fixture) stamp = Timestamp('2018-06-04 10:20:30', tz=tz) _datetime = datetime(2018, 6, 4, hour=10, From 7ba3649e542d9e175c105b3cffb79a0cbd879ee9 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Nov 2018 18:46:11 -0800 Subject: [PATCH 02/19] convert 'UTC' to UTC const --- pandas/_libs/tslib.pyx | 5 +++-- pandas/_libs/tslibs/conversion.pyx | 15 +++++++-------- pandas/_libs/tslibs/offsets.pyx | 3 ++- pandas/_libs/tslibs/timestamps.pyx | 9 +++++---- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index e346eb7e598ed..c5bc969ede3c9 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -33,6 +33,7 @@ from tslibs.parsing import parse_datetime_string from tslibs.timedeltas cimport cast_from_unit from tslibs.timezones cimport is_utc, is_tzlocal, get_dst_info +from tslibs.timezones import UTC from tslibs.conversion cimport (tz_convert_single, _TSObject, convert_datetime_to_tsobject, get_datetime64_nanos, @@ -211,7 +212,7 @@ def _test_parse_iso8601(object ts): check_dts_bounds(&obj.dts) if out_local == 1: obj.tzinfo = pytz.FixedOffset(out_tzoffset) - obj.value = tz_convert_single(obj.value, obj.tzinfo, 'UTC') + obj.value = tz_convert_single(obj.value, obj.tzinfo, UTC) return Timestamp(obj.value, tz=obj.tzinfo) else: return Timestamp(obj.value) @@ -673,7 +674,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', # dateutil.tz.tzoffset objects out_tzoffset_vals.add(out_tzoffset * 60.) tz = pytz.FixedOffset(out_tzoffset) - value = tz_convert_single(value, tz, 'UTC') + value = tz_convert_single(value, tz, UTC) else: # Add a marker for naive string, to track if we are # parsing mixed naive and aware strings diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 5a33f91ffca5a..925c30e8f4aa8 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -35,6 +35,7 @@ from timedeltas cimport cast_from_unit from timezones cimport (is_utc, is_tzlocal, is_fixed_offset, get_utcoffset, get_dst_info, get_timezone, maybe_get_tz, tz_compare) +from timezones import UTC from parsing import parse_datetime_string from nattype import nat_strings, NaT @@ -46,8 +47,6 @@ from nattype cimport NPY_NAT, checknull_with_nat NS_DTYPE = np.dtype('M8[ns]') TD_DTYPE = np.dtype('m8[ns]') -UTC = pytz.UTC - # ---------------------------------------------------------------------- # Misc Helpers @@ -442,7 +441,7 @@ cdef _TSObject convert_str_to_tsobject(object ts, object tz, object unit, check_dts_bounds(&obj.dts) if out_local == 1: obj.tzinfo = pytz.FixedOffset(out_tzoffset) - obj.value = tz_convert_single(obj.value, obj.tzinfo, 'UTC') + obj.value = tz_convert_single(obj.value, obj.tzinfo, UTC) if tz is None: check_dts_bounds(&obj.dts) check_overflows(obj) @@ -576,7 +575,7 @@ cdef inline datetime _localize_pydatetime(datetime dt, tzinfo tz): identically, i.e. discards nanos from Timestamps. It also assumes that the `tz` input is not None. """ - if tz == 'UTC' or tz is UTC: + if is_utc(tz): return UTC.localize(dt) try: # datetime.replace with pytz may be incorrect result @@ -603,7 +602,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): elif not PyDateTime_CheckExact(dt): # i.e. is a Timestamp return dt.tz_localize(tz) - elif tz == 'UTC' or tz is UTC: + elif is_utc(tz): return UTC.localize(dt) try: # datetime.replace with pytz may be incorrect result @@ -735,7 +734,7 @@ cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): int64_t arr[1] # See GH#17734 We should always be converting either from UTC or to UTC - assert (is_utc(tz1) or tz1 == 'UTC') or (is_utc(tz2) or tz2 == 'UTC') + assert is_utc(tz1) or is_utc(tz2) if val == NPY_NAT: return val @@ -743,13 +742,13 @@ cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): # Convert to UTC if is_tzlocal(tz1): utc_date = _tz_convert_tzlocal_utc(val, tz1, to_utc=True) - elif get_timezone(tz1) != 'UTC': + elif not is_utc(get_timezone(tz1)): arr[0] = val utc_date = _tz_convert_dst(arr, tz1, to_utc=True)[0] else: utc_date = val - if get_timezone(tz2) == 'UTC': + if is_utc(get_timezone(tz2)): return utc_date elif is_tzlocal(tz2): return _tz_convert_tzlocal_utc(utc_date, tz2, to_utc=False) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 7ef38cba0c37f..f3ac102bf177e 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -26,6 +26,7 @@ from conversion cimport tz_convert_single, pydt_to_i8, localize_pydatetime from nattype cimport NPY_NAT from np_datetime cimport (npy_datetimestruct, dtstruct_to_dt64, dt64_to_dtstruct) +from timezones import UTC # --------------------------------------------------------------------- # Constants @@ -211,7 +212,7 @@ def _to_dt64(dt, dtype='datetime64'): # Thus astype is needed to cast datetime to datetime64[D] if getattr(dt, 'tzinfo', None) is not None: i8 = pydt_to_i8(dt) - dt = tz_convert_single(i8, 'UTC', dt.tzinfo) + dt = tz_convert_single(i8, UTC, dt.tzinfo) dt = np.int64(dt).astype('datetime64[ns]') else: dt = np.datetime64(dt) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index f6a6257f92e7c..bb7a9a57b8a75 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -36,6 +36,7 @@ from timedeltas import Timedelta from timedeltas cimport delta_to_nanoseconds from timezones cimport ( get_timezone, is_utc, maybe_get_tz, treat_tz_as_pytz, tz_compare) +from timezones import UTC # ---------------------------------------------------------------------- # Constants @@ -416,7 +417,7 @@ cdef class _Timestamp(datetime): int64_t val val = self.value if self.tz is not None and not is_utc(self.tz): - val = tz_convert_single(self.value, 'UTC', self.tz) + val = tz_convert_single(self.value, UTC, self.tz) return val cpdef bint _get_start_end_field(self, str field): @@ -633,7 +634,7 @@ class Timestamp(_Timestamp): Return a new Timestamp representing UTC day and time. """ - return cls.now('UTC') + return cls.now(UTC) @classmethod def utcfromtimestamp(cls, ts): @@ -1108,7 +1109,7 @@ class Timestamp(_Timestamp): else: if tz is None: # reset tz - value = tz_convert_single(self.value, 'UTC', self.tz) + value = tz_convert_single(self.value, UTC, self.tz) return Timestamp(value, tz=None) else: raise TypeError('Cannot localize tz-aware Timestamp, use ' @@ -1178,7 +1179,7 @@ class Timestamp(_Timestamp): _tzinfo = self.tzinfo value = self.value if _tzinfo is not None: - value_tz = tz_convert_single(value, _tzinfo, 'UTC') + value_tz = tz_convert_single(value, _tzinfo, UTC) value += value - value_tz # setup components diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index c1f532d56304c..0d1e63f910dcf 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -613,7 +613,7 @@ def test_tz(self): assert conv.hour == 19 def test_utc_z_designator(self): - assert get_timezone(Timestamp('2014-11-02 01:00Z').tzinfo) == 'UTC' + assert get_timezone(Timestamp('2014-11-02 01:00Z').tzinfo) is utc def test_asm8(self): np.random.seed(7960929) From 5662f3a7aac56a72a56c8be54367b04a40ebdb78 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Nov 2018 19:22:38 -0800 Subject: [PATCH 03/19] Fix more occurances of 'UTC' --- pandas/_libs/tslibs/conversion.pyx | 7 ++++++- pandas/core/arrays/datetimes.py | 3 ++- pandas/tseries/frequencies.py | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index cee064abb113e..0446378fc6da2 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -641,7 +641,10 @@ cdef inline int64_t[:] _tz_convert_dst(int64_t[:] values, tzinfo tz, int64_t[:] deltas int64_t v - trans, deltas, typ = get_dst_info(tz) + if not is_tzlocal(tz): + # get_dst_info cannot extract offsets from tzlocal because its + # dependent on a datetime + trans, deltas, typ = get_dst_info(tz) if not to_utc: # We add `offset` below instead of subtracting it deltas = -1 * np.array(deltas, dtype='i8') @@ -650,6 +653,8 @@ cdef inline int64_t[:] _tz_convert_dst(int64_t[:] values, tzinfo tz, v = values[i] if v == NPY_NAT: result[i] = v + elif is_tzlocal(tz): + result[i] = _tz_convert_tzlocal_utc(v, tz, to_utc=to_utc) else: # TODO: Is it more efficient to call searchsorted pointwise or # on `values` outside the loop? We are not consistent about this. diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index c56e994e0ca2f..27ecf97cc1e62 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -797,7 +797,8 @@ def tz_localize(self, tz, ambiguous='raise', nonexistent='raise', if self.tz is not None: if tz is None: - new_dates = conversion.tz_convert(self.asi8, 'UTC', self.tz) + new_dates = conversion.tz_convert(self.asi8, timezones.UTC, + self.tz) else: raise TypeError("Already tz-aware, use tz_convert to convert.") else: diff --git a/pandas/tseries/frequencies.py b/pandas/tseries/frequencies.py index ac9a87b258056..62603db80489c 100644 --- a/pandas/tseries/frequencies.py +++ b/pandas/tseries/frequencies.py @@ -30,6 +30,7 @@ from pandas._libs.tslibs.resolution import Resolution from pandas._libs.tslibs.fields import build_field_sarray from pandas._libs.tslibs.conversion import tz_convert +from pandas._libs.tslibs.timezones import UTC from pandas._libs.algos import unique_deltas @@ -298,7 +299,7 @@ def __init__(self, index, warn=True): # the timezone so they are in local time if hasattr(index, 'tz'): if index.tz is not None: - self.values = tz_convert(self.values, 'UTC', index.tz) + self.values = tz_convert(self.values, UTC, index.tz) self.warn = warn From 28030419947f68892e1f87020e0e7734db1e49f7 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Nov 2018 20:52:28 -0800 Subject: [PATCH 04/19] Add notes --- pandas/_libs/tslibs/conversion.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 0446378fc6da2..86e0dcb275972 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -692,6 +692,7 @@ cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, datetime dt dt64_to_dtstruct(val, &dts) + # THIS NEEDS TO BE A LOCAL DATETIME! dt = datetime(dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us, tz) delta = int(get_utcoffset(tz, dt).total_seconds()) * 1000000000 @@ -790,7 +791,6 @@ cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, int64_t val if not is_utc(get_timezone(tz)): - #if get_timezone(tz) != 'UTC': converted = np.empty(n, dtype=np.int64) if is_tzlocal(tz): for i in range(n): From c5e11a62299f091b854587d8a3ab18c5da70a509 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Nov 2018 23:03:43 -0800 Subject: [PATCH 05/19] fix tzlocal with iteration --- pandas/_libs/tslibs/conversion.pyx | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 86e0dcb275972..2835e36b4f0e4 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -8,6 +8,7 @@ from numpy cimport int64_t, int32_t, ndarray cnp.import_array() import pytz +from dateutil.tz import tzutc # stdlib datetime imports from datetime import time as datetime_time @@ -539,7 +540,8 @@ cdef inline void localize_tso(_TSObject obj, tzinfo tz): elif obj.value == NPY_NAT: pass elif is_tzlocal(tz): - local_val = _tz_convert_tzlocal_utc(obj.value, tz, to_utc=False) + local_val = _tz_convert_tzlocal_utc(obj.value, tz, to_utc=False, + from_utc=True) dt64_to_dtstruct(local_val, &obj.dts) else: # Adjust datetime64 timestamp, recompute datetimestruct @@ -668,7 +670,8 @@ cdef inline int64_t[:] _tz_convert_dst(int64_t[:] values, tzinfo tz, cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, - bint to_utc=True): + bint to_utc=True, + bint from_utc=False): """ Convert the i8 representation of a datetime from a tzlocal timezone to UTC, or vice-versa. @@ -681,6 +684,8 @@ cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, tz : tzinfo to_utc : bint True if converting tzlocal _to_ UTC, False if going the other direction + from_utc: bint + True if val is a UTC timestamp, False if val is a wall/naive timestamp Returns ------- @@ -693,8 +698,12 @@ cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, dt64_to_dtstruct(val, &dts) # THIS NEEDS TO BE A LOCAL DATETIME! + # get_utcoffset (utcoffset udh) assumes the datetime is a wall time dt = datetime(dts.year, dts.month, dts.day, dts.hour, - dts.min, dts.sec, dts.us, tz) + dts.min, dts.sec, dts.us) + if from_utc: + dt = dt.replace(tzinfo=tzutc()) + dt = dt.astimezone(tz) delta = int(get_utcoffset(tz, dt).total_seconds()) * 1000000000 if not to_utc: @@ -713,7 +722,7 @@ cdef inline int64_t tz_convert_utc_to_tzlocal(int64_t utc_val, tzinfo tz): ------- local_val : int64_t """ - return _tz_convert_tzlocal_utc(utc_val, tz, to_utc=False) + return _tz_convert_tzlocal_utc(utc_val, tz, to_utc=False, from_utc=True) cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): From fbc50da50f421d4ddcdb85ee85bf04e78562325d Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Nov 2018 23:26:07 -0800 Subject: [PATCH 06/19] move codeblock --- pandas/_libs/tslibs/conversion.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 2835e36b4f0e4..2d8394078415f 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -647,9 +647,9 @@ cdef inline int64_t[:] _tz_convert_dst(int64_t[:] values, tzinfo tz, # get_dst_info cannot extract offsets from tzlocal because its # dependent on a datetime trans, deltas, typ = get_dst_info(tz) - if not to_utc: - # We add `offset` below instead of subtracting it - deltas = -1 * np.array(deltas, dtype='i8') + if not to_utc: + # We add `offset` below instead of subtracting it + deltas = -1 * np.array(deltas, dtype='i8') for i in range(n): v = values[i] From f351cd2e917432f800c1c91e8e692e8ff0280b4e Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Nov 2018 21:10:46 -0800 Subject: [PATCH 07/19] fix tz_localize(None) tzlocal() for Timestamp --- pandas/_libs/tslibs/conversion.pyx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 2d8394078415f..b4ea1a625ef15 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -697,10 +697,10 @@ cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, datetime dt dt64_to_dtstruct(val, &dts) - # THIS NEEDS TO BE A LOCAL DATETIME! - # get_utcoffset (utcoffset udh) assumes the datetime is a wall time dt = datetime(dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us) + # get_utcoffset (tz.utcoffset under the hood) only makes sense if datetime + # is _wall time_, so if val is a UTC timestamp convert to wall time if from_utc: dt = dt.replace(tzinfo=tzutc()) dt = dt.astimezone(tz) @@ -766,7 +766,8 @@ cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): if is_utc(get_timezone(tz2)): return utc_date elif is_tzlocal(tz2): - return _tz_convert_tzlocal_utc(utc_date, tz2, to_utc=False) + return _tz_convert_tzlocal_utc(utc_date, tz2, to_utc=False, + from_utc=True) else: # Convert UTC to other timezone arr[0] = utc_date From 5b97f0dfaa4823e2e66183d65ebce39ed078d6ac Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Nov 2018 21:22:36 -0800 Subject: [PATCH 08/19] Fix tz_localize(None) with tzlocal for DTI --- pandas/_libs/tslibs/conversion.pyx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index b4ea1a625ef15..4c240e26e3100 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -781,7 +781,7 @@ cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): @cython.boundscheck(False) @cython.wraparound(False) cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, - bint to_utc): + bint to_utc, bint from_utc=False): """ Convert the given values (in i8) either to UTC or from UTC. @@ -790,6 +790,9 @@ cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, vals : int64 ndarray tz1 : string / timezone object to_utc : bint + from_utc : bint + True if vals are UTC timestamps, False if vals are wall timestamps + Only applicable for tz=dateutil.tz.tzlocal() Returns ------- @@ -808,7 +811,8 @@ cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, if val == NPY_NAT: converted[i] = NPY_NAT else: - converted[i] = _tz_convert_tzlocal_utc(val, tz, to_utc) + converted[i] = _tz_convert_tzlocal_utc(val, tz, to_utc, + from_utc=from_utc) else: converted = _tz_convert_dst(vals, tz, to_utc) else: @@ -841,7 +845,8 @@ def tz_convert(int64_t[:] vals, object tz1, object tz2): # Convert to UTC utc_dates = _tz_convert_one_way(vals, tz1, to_utc=True) - converted = _tz_convert_one_way(utc_dates, tz2, to_utc=False) + converted = _tz_convert_one_way(utc_dates, tz2, to_utc=False, + from_utc=True) return np.array(converted, dtype=np.int64) From 81646883b8c40eb9f9e45a03b24854cd09d457a0 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Nov 2018 21:47:37 -0800 Subject: [PATCH 09/19] Add test for tzlocal and NaT --- pandas/tests/indexes/datetimes/test_construction.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 4b2c07af6af68..5c555b7c0ea66 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -5,6 +5,7 @@ import numpy as np import pytest import pytz +import dateutil from pandas._libs.tslib import OutOfBoundsDatetime from pandas._libs.tslibs import conversion @@ -527,6 +528,12 @@ def test_construction_with_tz_and_tz_aware_dti(self): with pytest.raises(TypeError): DatetimeIndex(dti, tz='Asia/Tokyo') + def test_construction_with_nat_and_tzlocal(self): + tz = dateutil.tz.tzlocal() + result = DatetimeIndex(['2018', 'NaT'], tz=tz) + expected = DatetimeIndex([Timestamp('2018', tz=tz), pd.NaT]) + tm.assert_index_equal(result, expected) + class TestTimeSeries(object): From 069e6e1c16a02e52b8ca106d607e814c8f5211e6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Nov 2018 21:57:38 -0800 Subject: [PATCH 10/19] Add whatsnew --- doc/source/whatsnew/v0.24.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 7d8ee975ba02c..c597dcee60e6d 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1248,6 +1248,8 @@ Timezones - Bug when indexing a :class:`Series` with a DST transition (:issue:`21846`) - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` where an ``AmbiguousTimeError`` or ``NonExistentTimeError`` would raise if a timezone aware timeseries ended on a DST transition (:issue:`19375`, :issue:`10117`) - Bug in :meth:`DataFrame.drop` and :meth:`Series.drop` when specifying a tz-aware Timestamp key to drop from a :class:`DatetimeIndex` with a DST transition (:issue:`21761`) +- Bug in :class:`DatetimeIndex` constructor where :class:`NaT` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:``) +- Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:``) Offsets ^^^^^^^ From de3c83bb946a7791d29a18db5f004228b24c9b02 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Nov 2018 22:10:41 -0800 Subject: [PATCH 11/19] Add issue number and one more fixedoffset --- doc/source/whatsnew/v0.24.0.rst | 4 ++-- pandas/conftest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index c597dcee60e6d..cde8b176ba244 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1248,8 +1248,8 @@ Timezones - Bug when indexing a :class:`Series` with a DST transition (:issue:`21846`) - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` where an ``AmbiguousTimeError`` or ``NonExistentTimeError`` would raise if a timezone aware timeseries ended on a DST transition (:issue:`19375`, :issue:`10117`) - Bug in :meth:`DataFrame.drop` and :meth:`Series.drop` when specifying a tz-aware Timestamp key to drop from a :class:`DatetimeIndex` with a DST transition (:issue:`21761`) -- Bug in :class:`DatetimeIndex` constructor where :class:`NaT` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:``) -- Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:``) +- Bug in :class:`DatetimeIndex` constructor where :class:`NaT` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:`23807`) +- Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:`23807`) Offsets ^^^^^^^ diff --git a/pandas/conftest.py b/pandas/conftest.py index 50bf2cf342fa3..3810828f60190 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -355,7 +355,7 @@ def unique_nulls_fixture(request): TIMEZONES = [None, 'UTC', 'US/Eastern', 'Asia/Tokyo', 'dateutil/US/Pacific', 'dateutil/Asia/Singapore', tzutc(), tzlocal(), FixedOffset(300), - FixedOffset(0)] + FixedOffset(0), FixedOffset(-300)] @td.parametrize_fixture_doc(str(TIMEZONES)) From fb6e96c566d655dc1abaa40dae81e3a6a53bb21d Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 20 Nov 2018 21:15:32 -0800 Subject: [PATCH 12/19] fix dateutil UTC converted to pytz UTC --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/_libs/tslibs/conversion.pyx | 2 +- pandas/tests/scalar/timestamp/test_timestamp.py | 6 ++++++ pandas/tests/tslibs/test_conversion.py | 9 +++++---- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 0df95bee6d30a..0c52b4aed3d19 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1251,6 +1251,7 @@ Timezones - Bug in :meth:`DataFrame.drop` and :meth:`Series.drop` when specifying a tz-aware Timestamp key to drop from a :class:`DatetimeIndex` with a DST transition (:issue:`21761`) - Bug in :class:`DatetimeIndex` constructor where :class:`NaT` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:`23807`) - Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:`23807`) +- Bug in :class:`Timestamp` constructor where a ``dateutil.tz.tzutc`` timezone passed with a ``datetime.datetime`` argument would be converted to a ``pytz.UTC`` timezone (:issue:`23807`) Offsets ^^^^^^^ diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 4c240e26e3100..f55f8cbd501d8 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -362,7 +362,7 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz, else: # UTC obj.value = pydatetime_to_dt64(ts, &obj.dts) - obj.tzinfo = pytz.utc + obj.tzinfo = tz else: obj.value = pydatetime_to_dt64(ts, &obj.dts) obj.tzinfo = ts.tzinfo diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index d24a79bfc63ca..8a834a45daf7c 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -589,6 +589,12 @@ def test_depreciate_tz_and_tzinfo_in_datetime_input(self, box): with tm.assert_produces_warning(FutureWarning): Timestamp(box(**kwargs), tz='US/Pacific') + def test_dont_convert_dateutil_utc_to_pytz_utc(self): + result = Timestamp(datetime(2018, 1, 1), tz=tzutc()) + expected = Timestamp(datetime(2018, 1, 1)).tz_localize(tzutc()) + assert result == expected + assert result.tz is tzutc() + class TestTimestamp(object): diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index de36c0bb2f789..fde1d1718a2a2 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -2,6 +2,7 @@ import numpy as np import pytest +from pytz import UTC from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones @@ -11,15 +12,15 @@ def compare_utc_to_local(tz_didx, utc_didx): - f = lambda x: conversion.tz_convert_single(x, 'UTC', tz_didx.tz) - result = conversion.tz_convert(tz_didx.asi8, 'UTC', tz_didx.tz) + f = lambda x: conversion.tz_convert_single(x, UTC, tz_didx.tz) + result = conversion.tz_convert(tz_didx.asi8, UTC, tz_didx.tz) result_single = np.vectorize(f)(tz_didx.asi8) tm.assert_numpy_array_equal(result, result_single) def compare_local_to_utc(tz_didx, utc_didx): - f = lambda x: conversion.tz_convert_single(x, tz_didx.tz, 'UTC') - result = conversion.tz_convert(utc_didx.asi8, tz_didx.tz, 'UTC') + f = lambda x: conversion.tz_convert_single(x, tz_didx.tz, UTC) + result = conversion.tz_convert(utc_didx.asi8, tz_didx.tz, UTC) result_single = np.vectorize(f)(utc_didx.asi8) tm.assert_numpy_array_equal(result, result_single) From 555afaafe3ee7c269499faec1de50b9dd8a648e6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 20 Nov 2018 22:00:53 -0800 Subject: [PATCH 13/19] Fix some tests --- pandas/tests/series/test_analytics.py | 2 +- pandas/tseries/offsets.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index a5a7cc2217864..e01d91661c2e6 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -338,7 +338,7 @@ def test_describe(self): def test_describe_with_tz(self, tz_naive_fixture): # GH 21332 tz = tz_naive_fixture - name = tz_naive_fixture + name = str(tz_naive_fixture) start = Timestamp(2018, 1, 1) end = Timestamp(2018, 1, 5) s = Series(date_range(start, end, tz=tz), name=name) diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index ca81b3bcfef2a..fedcf46277515 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -9,7 +9,7 @@ from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, Timedelta, Timestamp, ccalendar, conversion, delta_to_nanoseconds, frequencies as libfrequencies, normalize_date, - offsets as liboffsets) + offsets as liboffsets, timezones) from pandas._libs.tslibs.offsets import ( ApplyTypeError, BaseOffset, _get_calendar, _is_normalized, _to_dt64, apply_index_wraps, as_datetime, roll_yearday, shift_month) @@ -81,7 +81,7 @@ def wrapper(self, other): if result.tz is not None: # convert to UTC value = conversion.tz_convert_single( - result.value, 'UTC', result.tz) + result.value, timezones.UTC, result.tz) else: value = result.value result = Timestamp(value + nano) From 9857897c20e213549e8fb1fd1587339d147876c9 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 20 Nov 2018 23:27:24 -0800 Subject: [PATCH 14/19] don't assume pytz.UTC should localize a datetime --- pandas/_libs/tslibs/conversion.pyx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index f55f8cbd501d8..9b3a63e6e972e 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -577,8 +577,6 @@ cdef inline datetime _localize_pydatetime(datetime dt, tzinfo tz): identically, i.e. discards nanos from Timestamps. It also assumes that the `tz` input is not None. """ - if is_utc(tz): - return UTC.localize(dt) try: # datetime.replace with pytz may be incorrect result return tz.localize(dt) @@ -605,7 +603,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): # i.e. is a Timestamp return dt.tz_localize(tz) elif is_utc(tz): - return UTC.localize(dt) + return _localize_pydatetime(dt, tz) try: # datetime.replace with pytz may be incorrect result return tz.localize(dt) From c647ba052ea245f719d94e7e5daa17ca1bce6e03 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 21 Nov 2018 10:24:07 -0800 Subject: [PATCH 15/19] isort --- pandas/conftest.py | 8 ++++---- pandas/tests/indexes/datetimes/test_construction.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/conftest.py b/pandas/conftest.py index 3810828f60190..9c326c3d502d8 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -1,16 +1,16 @@ import importlib import os -from dateutil.tz import tzutc, tzlocal -import hypothesis -from hypothesis import strategies as st +from dateutil.tz import tzlocal, tzutc import numpy as np import pytest -from pytz import utc, FixedOffset +from pytz import FixedOffset, utc from pandas.compat import PY3 import pandas.util._test_decorators as td +import hypothesis +from hypothesis import strategies as st import pandas as pd hypothesis.settings.register_profile( diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 5c555b7c0ea66..02755c7e58a1d 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -2,10 +2,10 @@ from functools import partial from operator import attrgetter +import dateutil import numpy as np import pytest import pytz -import dateutil from pandas._libs.tslib import OutOfBoundsDatetime from pandas._libs.tslibs import conversion From c219ea3ed0aeb3c1ca6661ce34cc5699d9977c5f Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 21 Nov 2018 11:18:35 -0800 Subject: [PATCH 16/19] extract actual timezone object for DatetimeTZDtype --- pandas/core/dtypes/dtypes.py | 4 ++-- pandas/tests/indexes/datetimes/test_tools.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index 4dfefdec031b2..bd7c6630c7c5d 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -5,7 +5,7 @@ import numpy as np from pandas._libs.interval import Interval -from pandas._libs.tslibs import NaT, Period, Timestamp +from pandas._libs.tslibs import NaT, Period, Timestamp, timezones from pandas.core.dtypes.generic import ABCCategoricalIndex, ABCIndexClass @@ -516,7 +516,7 @@ def __new__(cls, unit=None, tz=None): m = cls._match.search(unit) if m is not None: unit = m.groupdict()['unit'] - tz = m.groupdict()['tz'] + tz = timezones.maybe_get_tz(m.groupdict()['tz']) except TypeError: raise ValueError("could not construct DatetimeTZDtype") diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index e31d63ff8c3b9..c24c1025ea63c 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -409,8 +409,8 @@ def test_to_datetime_tz_pytz(self, cache): hour=3, minute=0))], dtype=object) result = pd.to_datetime(arr, utc=True, cache=cache) - expected = DatetimeIndex(['2000-01-01 08:00:00', - '2000-06-01 07:00:00'], + expected = DatetimeIndex(['2000-01-01 08:00:00+00:00', + '2000-06-01 07:00:00+00:00'], dtype='datetime64[ns, UTC]', freq=None) tm.assert_index_equal(result, expected) @@ -488,8 +488,8 @@ def test_to_datetime_tz_psycopg2(self, cache): dtype=object) result = pd.to_datetime(arr, errors='coerce', utc=True, cache=cache) - expected = DatetimeIndex(['2000-01-01 08:00:00', - '2000-06-01 07:00:00'], + expected = DatetimeIndex(['2000-01-01 08:00:00+00:00', + '2000-06-01 07:00:00+00:00'], dtype='datetime64[ns, UTC]', freq=None) tm.assert_index_equal(result, expected) From 44a33ab2471ff838c14026f87c4acab2c54fcd1e Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 21 Nov 2018 11:37:23 -0800 Subject: [PATCH 17/19] from_utc -> not to_utc --- pandas/_libs/tslibs/conversion.pyx | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 9b3a63e6e972e..b4a13d3d0ada9 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -540,8 +540,7 @@ cdef inline void localize_tso(_TSObject obj, tzinfo tz): elif obj.value == NPY_NAT: pass elif is_tzlocal(tz): - local_val = _tz_convert_tzlocal_utc(obj.value, tz, to_utc=False, - from_utc=True) + local_val = _tz_convert_tzlocal_utc(obj.value, tz, to_utc=False) dt64_to_dtstruct(local_val, &obj.dts) else: # Adjust datetime64 timestamp, recompute datetimestruct @@ -668,8 +667,7 @@ cdef inline int64_t[:] _tz_convert_dst(int64_t[:] values, tzinfo tz, cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, - bint to_utc=True, - bint from_utc=False): + bint to_utc=True): """ Convert the i8 representation of a datetime from a tzlocal timezone to UTC, or vice-versa. @@ -682,8 +680,6 @@ cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, tz : tzinfo to_utc : bint True if converting tzlocal _to_ UTC, False if going the other direction - from_utc: bint - True if val is a UTC timestamp, False if val is a wall/naive timestamp Returns ------- @@ -699,7 +695,7 @@ cdef inline int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, dts.min, dts.sec, dts.us) # get_utcoffset (tz.utcoffset under the hood) only makes sense if datetime # is _wall time_, so if val is a UTC timestamp convert to wall time - if from_utc: + if not to_utc: dt = dt.replace(tzinfo=tzutc()) dt = dt.astimezone(tz) delta = int(get_utcoffset(tz, dt).total_seconds()) * 1000000000 @@ -720,7 +716,7 @@ cdef inline int64_t tz_convert_utc_to_tzlocal(int64_t utc_val, tzinfo tz): ------- local_val : int64_t """ - return _tz_convert_tzlocal_utc(utc_val, tz, to_utc=False, from_utc=True) + return _tz_convert_tzlocal_utc(utc_val, tz, to_utc=False) cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): @@ -764,8 +760,7 @@ cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): if is_utc(get_timezone(tz2)): return utc_date elif is_tzlocal(tz2): - return _tz_convert_tzlocal_utc(utc_date, tz2, to_utc=False, - from_utc=True) + return _tz_convert_tzlocal_utc(utc_date, tz2, to_utc=False) else: # Convert UTC to other timezone arr[0] = utc_date @@ -779,7 +774,7 @@ cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): @cython.boundscheck(False) @cython.wraparound(False) cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, - bint to_utc, bint from_utc=False): + bint to_utc): """ Convert the given values (in i8) either to UTC or from UTC. @@ -788,9 +783,6 @@ cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, vals : int64 ndarray tz1 : string / timezone object to_utc : bint - from_utc : bint - True if vals are UTC timestamps, False if vals are wall timestamps - Only applicable for tz=dateutil.tz.tzlocal() Returns ------- @@ -809,8 +801,7 @@ cdef inline int64_t[:] _tz_convert_one_way(int64_t[:] vals, object tz, if val == NPY_NAT: converted[i] = NPY_NAT else: - converted[i] = _tz_convert_tzlocal_utc(val, tz, to_utc, - from_utc=from_utc) + converted[i] = _tz_convert_tzlocal_utc(val, tz, to_utc) else: converted = _tz_convert_dst(vals, tz, to_utc) else: @@ -843,8 +834,7 @@ def tz_convert(int64_t[:] vals, object tz1, object tz2): # Convert to UTC utc_dates = _tz_convert_one_way(vals, tz1, to_utc=True) - converted = _tz_convert_one_way(utc_dates, tz2, to_utc=False, - from_utc=True) + converted = _tz_convert_one_way(utc_dates, tz2, to_utc=False) return np.array(converted, dtype=np.int64) From 9828aa502e15d110bc88af050bfe484ecfa5c20b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 21 Nov 2018 12:35:42 -0800 Subject: [PATCH 18/19] Assert already checks timezone --- pandas/tests/scalar/timestamp/test_timestamp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 8a834a45daf7c..b2c05d1564a48 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -593,7 +593,6 @@ def test_dont_convert_dateutil_utc_to_pytz_utc(self): result = Timestamp(datetime(2018, 1, 1), tz=tzutc()) expected = Timestamp(datetime(2018, 1, 1)).tz_localize(tzutc()) assert result == expected - assert result.tz is tzutc() class TestTimestamp(object): From 15ba5fb8bb462672d2153e2b4e0db4146a2a58d9 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 21 Nov 2018 18:48:27 -0800 Subject: [PATCH 19/19] isort --- pandas/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/conftest.py b/pandas/conftest.py index 9c326c3d502d8..f450193d9388e 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -2,6 +2,8 @@ import os from dateutil.tz import tzlocal, tzutc +import hypothesis +from hypothesis import strategies as st import numpy as np import pytest from pytz import FixedOffset, utc @@ -9,8 +11,6 @@ from pandas.compat import PY3 import pandas.util._test_decorators as td -import hypothesis -from hypothesis import strategies as st import pandas as pd hypothesis.settings.register_profile(