From 81e8987c083abbc99628f3faa26468917f72bd2b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 29 Nov 2018 21:17:26 -0800 Subject: [PATCH 1/5] cdef NaT constant --- pandas/_libs/tslib.pyx | 4 +- pandas/_libs/tslibs/conversion.pyx | 4 +- pandas/_libs/tslibs/nattype.pxd | 11 +++++ pandas/_libs/tslibs/nattype.pyx | 75 +++++++++++++++--------------- pandas/_libs/tslibs/timestamps.pyx | 3 +- pandas/_libs/tslibs/timezones.pyx | 6 ++- pandas/core/arrays/datetimes.py | 2 +- pandas/core/arrays/period.py | 3 +- 8 files changed, 60 insertions(+), 48 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 609608a0948c5..6f54818a9b253 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -40,8 +40,8 @@ from tslibs.conversion cimport (tz_convert_single, _TSObject, tz_convert_utc_to_tzlocal) # many modules still look for NaT and iNaT here despite them not being needed -from tslibs.nattype import nat_strings, NaT, iNaT # noqa:F821 -from tslibs.nattype cimport checknull_with_nat, NPY_NAT +from tslibs.nattype import nat_strings, iNaT # noqa:F821 +from tslibs.nattype cimport checknull_with_nat, NPY_NAT, NAT as NaT from tslibs.offsets cimport to_offset diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 67c2793e4bcef..514a0b3d5de11 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -39,8 +39,8 @@ from timezones cimport (is_utc, is_tzlocal, is_fixed_offset, from timezones import UTC from parsing import parse_datetime_string -from nattype import nat_strings, NaT -from nattype cimport NPY_NAT, checknull_with_nat +from nattype import nat_strings +from nattype cimport NPY_NAT, checknull_with_nat, NAT as NaT # ---------------------------------------------------------------------- # Constants diff --git a/pandas/_libs/tslibs/nattype.pxd b/pandas/_libs/tslibs/nattype.pxd index 382ac9d323918..1eb5ec25c8e1a 100644 --- a/pandas/_libs/tslibs/nattype.pxd +++ b/pandas/_libs/tslibs/nattype.pxd @@ -1,9 +1,20 @@ # -*- coding: utf-8 -*- +from cpython.datetime cimport datetime + from numpy cimport int64_t cdef int64_t NPY_NAT cdef bint _nat_scalar_rules[6] + +cdef class _NaT(datetime): + cdef readonly: + int64_t value + object freq + +cdef _NaT NAT + + cdef bint checknull_with_nat(object val) cdef bint is_null_datetimelike(object val) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 7b7f5f2e34c5f..b59312e97f235 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -47,7 +47,7 @@ def _make_nan_func(func_name, doc): def _make_nat_func(func_name, doc): def f(*args, **kwargs): - return NaT + return NAT f.__name__ = func_name f.__doc__ = doc return f @@ -67,10 +67,10 @@ def _make_error_func(func_name, cls): cdef _nat_divide_op(self, other): - if PyDelta_Check(other) or is_timedelta64_object(other) or other is NaT: + if PyDelta_Check(other) or is_timedelta64_object(other) or other is NAT: return np.nan if is_integer_object(other) or is_float_object(other): - return NaT + return NAT return NotImplemented @@ -82,15 +82,15 @@ cdef _nat_rdivide_op(self, other): def __nat_unpickle(*args): # return constant defined in the module - return NaT + return NAT # ---------------------------------------------------------------------- cdef class _NaT(datetime): - cdef readonly: - int64_t value - object freq + # cdef readonly: + # int64_t value + # object freq def __hash__(_NaT self): # py3k needs this defined here @@ -116,18 +116,18 @@ cdef class _NaT(datetime): def __add__(self, other): if PyDateTime_Check(other): - return NaT + return NAT elif hasattr(other, 'delta'): # Timedelta, offsets.Tick, offsets.Week - return NaT + return NAT elif getattr(other, '_typ', None) in ['dateoffset', 'series', 'period', 'datetimeindex', 'timedeltaindex']: # Duplicate logic in _Timestamp.__add__ to avoid needing # to subclass; allows us to @final(_Timestamp.__add__) return NotImplemented - return NaT + return NAT def __sub__(self, other): # Duplicate some logic from _Timestamp.__sub__ to avoid needing @@ -184,19 +184,6 @@ cdef class _NaT(datetime): """ Returns a numpy.datetime64 object with 'ns' precision """ return np.datetime64('NaT', 'ns') - -class NaTType(_NaT): - """(N)ot-(A)-(T)ime, the time equivalent of NaN""" - - def __new__(cls): - cdef _NaT base - - base = _NaT.__new__(cls, 1, 1, 1) - base.value = NPY_NAT - base.freq = None - - return base - def __repr__(self): return 'NaT' @@ -216,20 +203,11 @@ class NaTType(_NaT): def __long__(self): return NPY_NAT - def __reduce_ex__(self, protocol): - # python 3.6 compat - # http://bugs.python.org/issue28730 - # now __reduce_ex__ is defined and higher priority than __reduce__ - return self.__reduce__() - - def __reduce__(self): - return (__nat_unpickle, (None, )) - def total_seconds(self): """ Total duration of timedelta in seconds (to ns precision) """ - # GH 10939 + # GH#10939 return np.nan @property @@ -260,6 +238,28 @@ class NaTType(_NaT): def is_year_end(self): return False + +class NaTType(_NaT): + """(N)ot-(A)-(T)ime, the time equivalent of NaN""" + + def __new__(cls): + cdef _NaT base + + base = _NaT.__new__(cls, 1, 1, 1) + base.value = NPY_NAT + base.freq = None + + return base + + def __reduce_ex__(self, protocol): + # python 3.6 compat + # http://bugs.python.org/issue28730 + # now __reduce_ex__ is defined and higher priority than __reduce__ + return self.__reduce__() + + def __reduce__(self): + return (__nat_unpickle, (None, )) + def __rdiv__(self, other): return _nat_rdivide_op(self, other) @@ -271,7 +271,7 @@ class NaTType(_NaT): def __rmul__(self, other): if is_integer_object(other) or is_float_object(other): - return NaT + return NAT return NotImplemented # ---------------------------------------------------------------------- @@ -659,14 +659,15 @@ class NaTType(_NaT): """) -NaT = NaTType() +NAT = NaTType() # C-visible +NaT = NAT # Python-visible # ---------------------------------------------------------------------- cdef inline bint checknull_with_nat(object val): """ utility to check if a value is a nat or not """ - return val is None or util.is_nan(val) or val is NaT + return val is None or util.is_nan(val) or val is NAT cdef inline bint is_null_datetimelike(object val): @@ -683,7 +684,7 @@ cdef inline bint is_null_datetimelike(object val): """ if val is None or util.is_nan(val): return True - elif val is NaT: + elif val is NAT: return True elif util.is_timedelta64_object(val): return val.view('int64') == NPY_NAT diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index f414cd161e562..f89fe2dca07e5 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -26,8 +26,7 @@ from conversion import tz_localize_to_utc, normalize_i8_timestamps from conversion cimport (tz_convert_single, _TSObject, convert_to_tsobject, convert_datetime_to_tsobject) from fields import get_start_end_field, get_date_name_field -from nattype import NaT -from nattype cimport NPY_NAT +from nattype cimport NPY_NAT, NAT as NaT from np_datetime import OutOfBoundsDatetime from np_datetime cimport (reverse_ops, cmp_scalar, check_dts_bounds, npy_datetimestruct, dt64_to_dtstruct) diff --git a/pandas/_libs/tslibs/timezones.pyx b/pandas/_libs/tslibs/timezones.pyx index 5fa8a45af3083..57fa357d26fce 100644 --- a/pandas/_libs/tslibs/timezones.pyx +++ b/pandas/_libs/tslibs/timezones.pyx @@ -2,6 +2,8 @@ from cython import Py_ssize_t +from cpython.datetime cimport tzinfo + # dateutil compat from dateutil.tz import ( tzutc as _dateutil_tzutc, @@ -35,12 +37,12 @@ cdef inline bint is_tzlocal(object tz): return isinstance(tz, _dateutil_tzlocal) -cdef inline bint treat_tz_as_pytz(object tz): +cdef inline bint treat_tz_as_pytz(tzinfo tz): return (hasattr(tz, '_utc_transition_times') and hasattr(tz, '_transition_info')) -cdef inline bint treat_tz_as_dateutil(object tz): +cdef inline bint treat_tz_as_dateutil(tzinfo tz): return hasattr(tz, '_trans_list') and hasattr(tz, '_trans_idx') diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 4d3caaacca1c1..e3471b580bdad 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -6,8 +6,8 @@ from pytz import utc from pandas._libs import lib, tslib -from pandas._libs.tslib import NaT, Timestamp, iNaT from pandas._libs.tslibs import ( + NaT, Timestamp, iNaT, ccalendar, conversion, fields, normalize_date, resolution as libresolution, timezones) import pandas.compat as compat diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 53629dca4d391..bb009d9370dec 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -4,8 +4,7 @@ import numpy as np -from pandas._libs.tslib import NaT, iNaT -from pandas._libs.tslibs import period as libperiod +from pandas._libs.tslibs import NaT, iNaT, period as libperiod from pandas._libs.tslibs.fields import isleapyear_arr from pandas._libs.tslibs.period import ( DIFFERENT_FREQ_INDEX, IncompatibleFrequency, Period, get_period_field_arr, From de3bc062b647f706b099dc9ee161a364806298a8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 30 Nov 2018 16:04:35 -0800 Subject: [PATCH 2/5] revert mixup --- pandas/_libs/tslibs/timezones.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/timezones.pyx b/pandas/_libs/tslibs/timezones.pyx index 57fa357d26fce..9f8922b274abd 100644 --- a/pandas/_libs/tslibs/timezones.pyx +++ b/pandas/_libs/tslibs/timezones.pyx @@ -37,12 +37,12 @@ cdef inline bint is_tzlocal(object tz): return isinstance(tz, _dateutil_tzlocal) -cdef inline bint treat_tz_as_pytz(tzinfo tz): +cdef inline bint treat_tz_as_pytz(object tz): return (hasattr(tz, '_utc_transition_times') and hasattr(tz, '_transition_info')) -cdef inline bint treat_tz_as_dateutil(tzinfo tz): +cdef inline bint treat_tz_as_dateutil(object tz): return hasattr(tz, '_trans_list') and hasattr(tz, '_trans_idx') From 03fc288a1bad67592c260415322b87feb698d157 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 30 Nov 2018 16:06:41 -0800 Subject: [PATCH 3/5] update imports --- pandas/tests/scalar/test_nat.py | 2 +- pandas/tests/scalar/timedelta/test_timedelta.py | 2 +- pandas/tests/tseries/offsets/test_offsets.py | 8 +++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pandas/tests/scalar/test_nat.py b/pandas/tests/scalar/test_nat.py index ddf3984744114..d2a31de5c0938 100644 --- a/pandas/tests/scalar/test_nat.py +++ b/pandas/tests/scalar/test_nat.py @@ -4,7 +4,7 @@ import pytest import pytz -from pandas._libs.tslib import iNaT +from pandas._libs.tslibs import iNaT from pandas import ( DatetimeIndex, Index, NaT, Period, Series, Timedelta, TimedeltaIndex, diff --git a/pandas/tests/scalar/timedelta/test_timedelta.py b/pandas/tests/scalar/timedelta/test_timedelta.py index 82cfdb9e0751e..477c8aa4c3b0d 100644 --- a/pandas/tests/scalar/timedelta/test_timedelta.py +++ b/pandas/tests/scalar/timedelta/test_timedelta.py @@ -4,7 +4,7 @@ import numpy as np import pytest -from pandas._libs.tslib import NaT, iNaT +from pandas._libs.tslibs import NaT, iNaT import pandas.compat as compat import pandas as pd diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 269b017fa2141..e3e1573dcc19e 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -5,13 +5,11 @@ import pytest import pytz -import pandas._libs.tslib as tslib -from pandas._libs.tslib import NaT, Timestamp -from pandas._libs.tslibs import conversion, timezones +from pandas._libs.tslibs import ( + conversion, timezones, NaT, Timestamp, Timedelta, OutOfBoundsDatetime) from pandas._libs.tslibs.frequencies import ( INVALID_FREQ_ERR_MSG, get_freq_code, get_freq_str) import pandas._libs.tslibs.offsets as liboffsets -from pandas._libs.tslibs.timedeltas import Timedelta import pandas.compat as compat from pandas.compat import range from pandas.compat.numpy import np_datetime64_compat @@ -124,7 +122,7 @@ def test_apply_out_of_range(self, tz_naive_fixture): assert isinstance(result, datetime) assert t.tzinfo == result.tzinfo - except tslib.OutOfBoundsDatetime: + except OutOfBoundsDatetime: raise except (ValueError, KeyError): # we are creating an invalid offset From 6e8c0eebcf9f581f271100773e205fab5f19c425 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 1 Dec 2018 19:35:16 -0800 Subject: [PATCH 4/5] isort --- pandas/core/arrays/datetimes.py | 5 ++--- pandas/tests/tseries/offsets/test_offsets.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index e3471b580bdad..a6ba3ef0f687e 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -7,9 +7,8 @@ from pandas._libs import lib, tslib from pandas._libs.tslibs import ( - NaT, Timestamp, iNaT, - ccalendar, conversion, fields, normalize_date, resolution as libresolution, - timezones) + NaT, Timestamp, ccalendar, conversion, fields, iNaT, normalize_date, + resolution as libresolution, timezones) import pandas.compat as compat from pandas.errors import PerformanceWarning from pandas.util._decorators import Appender, cache_readonly diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index e3e1573dcc19e..030887ac731f3 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -6,7 +6,7 @@ import pytz from pandas._libs.tslibs import ( - conversion, timezones, NaT, Timestamp, Timedelta, OutOfBoundsDatetime) + NaT, OutOfBoundsDatetime, Timedelta, Timestamp, conversion, timezones) from pandas._libs.tslibs.frequencies import ( INVALID_FREQ_ERR_MSG, get_freq_code, get_freq_str) import pandas._libs.tslibs.offsets as liboffsets From 05c2ce0df64cad5900c46822e11ea7bfeba5976b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 2 Dec 2018 09:25:46 -0800 Subject: [PATCH 5/5] change NAT -> c_NaT --- pandas/_libs/tslib.pyx | 2 +- pandas/_libs/tslibs/conversion.pyx | 2 +- pandas/_libs/tslibs/nattype.pxd | 2 +- pandas/_libs/tslibs/nattype.pyx | 24 ++++++++++++------------ pandas/_libs/tslibs/period.pyx | 5 +++-- pandas/_libs/tslibs/timedeltas.pyx | 4 ++-- pandas/_libs/tslibs/timestamps.pyx | 2 +- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 2dca903655068..5528e183af6e6 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -41,7 +41,7 @@ from tslibs.conversion cimport (tz_convert_single, _TSObject, # many modules still look for NaT and iNaT here despite them not being needed from tslibs.nattype import nat_strings, iNaT # noqa:F821 -from tslibs.nattype cimport checknull_with_nat, NPY_NAT, NAT as NaT +from tslibs.nattype cimport checknull_with_nat, NPY_NAT, c_NaT as NaT from tslibs.offsets cimport to_offset diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 96f21d242390b..e6e7884f05b20 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -40,7 +40,7 @@ from timezones import UTC from parsing import parse_datetime_string from nattype import nat_strings -from nattype cimport NPY_NAT, checknull_with_nat, NAT as NaT +from nattype cimport NPY_NAT, checknull_with_nat, c_NaT as NaT # ---------------------------------------------------------------------- # Constants diff --git a/pandas/_libs/tslibs/nattype.pxd b/pandas/_libs/tslibs/nattype.pxd index 1eb5ec25c8e1a..f649518e969be 100644 --- a/pandas/_libs/tslibs/nattype.pxd +++ b/pandas/_libs/tslibs/nattype.pxd @@ -13,7 +13,7 @@ cdef class _NaT(datetime): int64_t value object freq -cdef _NaT NAT +cdef _NaT c_NaT cdef bint checknull_with_nat(object val) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index b59312e97f235..42ec235992089 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -47,7 +47,7 @@ def _make_nan_func(func_name, doc): def _make_nat_func(func_name, doc): def f(*args, **kwargs): - return NAT + return c_NaT f.__name__ = func_name f.__doc__ = doc return f @@ -67,10 +67,10 @@ def _make_error_func(func_name, cls): cdef _nat_divide_op(self, other): - if PyDelta_Check(other) or is_timedelta64_object(other) or other is NAT: + if PyDelta_Check(other) or is_timedelta64_object(other) or other is c_NaT: return np.nan if is_integer_object(other) or is_float_object(other): - return NAT + return c_NaT return NotImplemented @@ -82,7 +82,7 @@ cdef _nat_rdivide_op(self, other): def __nat_unpickle(*args): # return constant defined in the module - return NAT + return c_NaT # ---------------------------------------------------------------------- @@ -116,18 +116,18 @@ cdef class _NaT(datetime): def __add__(self, other): if PyDateTime_Check(other): - return NAT + return c_NaT elif hasattr(other, 'delta'): # Timedelta, offsets.Tick, offsets.Week - return NAT + return c_NaT elif getattr(other, '_typ', None) in ['dateoffset', 'series', 'period', 'datetimeindex', 'timedeltaindex']: # Duplicate logic in _Timestamp.__add__ to avoid needing # to subclass; allows us to @final(_Timestamp.__add__) return NotImplemented - return NAT + return c_NaT def __sub__(self, other): # Duplicate some logic from _Timestamp.__sub__ to avoid needing @@ -271,7 +271,7 @@ class NaTType(_NaT): def __rmul__(self, other): if is_integer_object(other) or is_float_object(other): - return NAT + return c_NaT return NotImplemented # ---------------------------------------------------------------------- @@ -659,15 +659,15 @@ class NaTType(_NaT): """) -NAT = NaTType() # C-visible -NaT = NAT # Python-visible +c_NaT = NaTType() # C-visible +NaT = c_NaT # Python-visible # ---------------------------------------------------------------------- cdef inline bint checknull_with_nat(object val): """ utility to check if a value is a nat or not """ - return val is None or util.is_nan(val) or val is NAT + return val is None or util.is_nan(val) or val is c_NaT cdef inline bint is_null_datetimelike(object val): @@ -684,7 +684,7 @@ cdef inline bint is_null_datetimelike(object val): """ if val is None or util.is_nan(val): return True - elif val is NAT: + elif val is c_NaT: return True elif util.is_timedelta64_object(val): return val.view('int64') == NPY_NAT diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index e02e493c32a00..dfbf24cf177f6 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -46,8 +46,9 @@ from frequencies cimport (get_freq_code, get_base_alias, get_rule_month) from parsing import parse_time_string from resolution import Resolution -from nattype import nat_strings, NaT -from nattype cimport _nat_scalar_rules, NPY_NAT, is_null_datetimelike +from nattype import nat_strings +from nattype cimport ( + _nat_scalar_rules, NPY_NAT, is_null_datetimelike, c_NaT as NaT) from offsets cimport to_offset from offsets import _Tick diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 4d612a6f43107..b0bead2f66ce4 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -33,8 +33,8 @@ from ccalendar import DAY_SECONDS from np_datetime cimport (cmp_scalar, reverse_ops, td64_to_tdstruct, pandas_timedeltastruct) -from nattype import nat_strings, NaT -from nattype cimport checknull_with_nat, NPY_NAT +from nattype import nat_strings +from nattype cimport checknull_with_nat, NPY_NAT, c_NaT as NaT from offsets cimport to_offset # ---------------------------------------------------------------------- diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index f89fe2dca07e5..b4862a5f3b02f 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -26,7 +26,7 @@ from conversion import tz_localize_to_utc, normalize_i8_timestamps from conversion cimport (tz_convert_single, _TSObject, convert_to_tsobject, convert_datetime_to_tsobject) from fields import get_start_end_field, get_date_name_field -from nattype cimport NPY_NAT, NAT as NaT +from nattype cimport NPY_NAT, c_NaT as NaT from np_datetime import OutOfBoundsDatetime from np_datetime cimport (reverse_ops, cmp_scalar, check_dts_bounds, npy_datetimestruct, dt64_to_dtstruct)