From 53ec4d138e769b152914e66402ba02cf1b7c542a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 6 Oct 2017 09:24:17 -0700 Subject: [PATCH 01/13] implement npy_dtime.pyx --- pandas/_libs/tslib.pyx | 18 ++------ pandas/_libs/tslibs/npy_dtime.pxd | 16 +++++++ pandas/_libs/tslibs/npy_dtime.pyx | 77 +++++++++++++++++++++++++++++++ pandas/_libs/tslibs/strptime.pyx | 21 +++------ setup.py | 5 ++ 5 files changed, 108 insertions(+), 29 deletions(-) create mode 100644 pandas/_libs/tslibs/npy_dtime.pxd create mode 100644 pandas/_libs/tslibs/npy_dtime.pyx diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index f58aaa0ce3234..69a7af53a96f1 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -46,7 +46,6 @@ from datetime cimport ( npy_datetime, is_leapyear, dayofweek, - check_dts_bounds, PANDAS_FR_ns, PyDateTime_Check, PyDate_Check, PyDateTime_IMPORT, @@ -57,6 +56,9 @@ from datetime cimport ( from datetime import timedelta, datetime from datetime import time as datetime_time +from tslibs.npy_dtime cimport check_dts_bounds as _check_dts_bounds +from tslibs.npy_dtime import OutOfBoundsDatetime + from khash cimport ( khiter_t, kh_destroy_int64, kh_put_int64, @@ -1825,18 +1827,6 @@ cpdef inline object _localize_pydatetime(object dt, object tz): return dt.replace(tzinfo=tz) -class OutOfBoundsDatetime(ValueError): - pass - -cdef inline _check_dts_bounds(pandas_datetimestruct *dts): - if check_dts_bounds(dts): - fmt = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (dts.year, dts.month, - dts.day, dts.hour, - dts.min, dts.sec) - raise OutOfBoundsDatetime( - 'Out of bounds nanosecond timestamp: %s' % fmt) - - def datetime_to_datetime64(ndarray[object] values): cdef: Py_ssize_t i, n = len(values) @@ -1959,8 +1949,6 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None, # const for parsers -_DEFAULT_DATETIME = datetime(1, 1, 1).replace( - hour=0, minute=0, second=0, microsecond=0) _MONTHS = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC'] _MONTH_NUMBERS = {k: i for i, k in enumerate(_MONTHS)} diff --git a/pandas/_libs/tslibs/npy_dtime.pxd b/pandas/_libs/tslibs/npy_dtime.pxd new file mode 100644 index 0000000000000..d4079aae68900 --- /dev/null +++ b/pandas/_libs/tslibs/npy_dtime.pxd @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# cython: profile=False + +from numpy cimport int64_t, int32_t + + +cdef extern from "../src/datetime/np_datetime.h": + ctypedef struct pandas_datetimestruct: + int64_t year + int32_t month, day, hour, min, sec, us, ps, as + + +cdef check_dts_bounds(pandas_datetimestruct *dts) + +cdef int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil +cdef void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil diff --git a/pandas/_libs/tslibs/npy_dtime.pyx b/pandas/_libs/tslibs/npy_dtime.pyx new file mode 100644 index 0000000000000..e802566d823ad --- /dev/null +++ b/pandas/_libs/tslibs/npy_dtime.pyx @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# cython: profile=False + +from numpy cimport int64_t + +cdef extern from "numpy/ndarrayobject.h": + ctypedef int64_t npy_timedelta + ctypedef int64_t npy_datetime + + +cdef extern from "../src/datetime/np_datetime.h": + ctypedef enum PANDAS_DATETIMEUNIT: + PANDAS_FR_Y + PANDAS_FR_M + PANDAS_FR_W + PANDAS_FR_D + PANDAS_FR_B + PANDAS_FR_h + PANDAS_FR_m + PANDAS_FR_s + PANDAS_FR_ms + PANDAS_FR_us + PANDAS_FR_ns + PANDAS_FR_ps + PANDAS_FR_fs + PANDAS_FR_as + + int cmp_pandas_datetimestruct(pandas_datetimestruct *a, + pandas_datetimestruct *b) + + npy_datetime pandas_datetimestruct_to_datetime(PANDAS_DATETIMEUNIT fr, + pandas_datetimestruct *d) nogil + void pandas_datetime_to_datetimestruct(npy_datetime val, + PANDAS_DATETIMEUNIT fr, + pandas_datetimestruct *result) nogil + pandas_datetimestruct _NS_MIN_DTS, _NS_MAX_DTS + +# ---------------------------------------------------------------------- + +class OutOfBoundsDatetime(ValueError): + pass + + +cdef inline check_dts_bounds(pandas_datetimestruct *dts): + """Returns True if an error needs to be raised""" + cdef: + bint error = False + + if (dts.year <= 1677 and + cmp_pandas_datetimestruct(dts, &_NS_MIN_DTS) == -1): + error = True + elif (dts.year >= 2262 and + cmp_pandas_datetimestruct(dts, &_NS_MAX_DTS) == 1): + error = True + + if error: + fmt = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (dts.year, dts.month, + dts.day, dts.hour, + dts.min, dts.sec) + raise OutOfBoundsDatetime( + 'Out of bounds nanosecond timestamp: %s' % fmt) + + +# ---------------------------------------------------------------------- +# Conversion + +cdef inline int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil: + """Convenience function to call pandas_datetimestruct_to_datetime + with the by-far-most-common frequency PANDAS_FR_ns""" + return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts) + + +cdef inline void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil: + """Convenience function to call pandas_datetime_to_datetimestruct + with the by-far-most-common frequency PANDAS_FR_ns""" + pandas_datetime_to_datetimestruct(dt64, PANDAS_FR_ns, out) + return diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index 20b24d6be9a58..012c4d1f093b7 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -33,12 +33,8 @@ from numpy cimport ndarray, int64_t from datetime import date as datetime_date from datetime cimport datetime -# This is src/datetime.pxd -from datetime cimport ( - PANDAS_FR_ns, - check_dts_bounds, - pandas_datetimestruct, - pandas_datetimestruct_to_datetime) +from npy_dtime cimport (check_dts_bounds, + dtstruct_to_dt64, pandas_datetimestruct) from util cimport is_string_object, get_nat @@ -323,18 +319,15 @@ def array_strptime(ndarray[object] values, object fmt, dts.us = us dts.ps = ns * 1000 - iresult[i] = pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts) - if check_dts_bounds(&dts): + iresult[i] = dtstruct_to_dt64(&dts) + try: + check_dts_bounds(&dts): + except ValueError: if is_coerce: iresult[i] = NPY_NAT continue else: - from pandas._libs.tslib import OutOfBoundsDatetime - fmt = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (dts.year, dts.month, - dts.day, dts.hour, - dts.min, dts.sec) - raise OutOfBoundsDatetime( - 'Out of bounds nanosecond timestamp: %s' % fmt) + raise return result diff --git a/setup.py b/setup.py index 365d387dc54d6..ee4ce869cd041 100755 --- a/setup.py +++ b/setup.py @@ -342,6 +342,7 @@ class CheckSDist(sdist_class): 'pandas/_libs/sparse.pyx', 'pandas/_libs/parsers.pyx', 'pandas/_libs/tslibs/strptime.pyx', + 'pandas/_libs/tslibs/npy_dtime.pyx', 'pandas/_libs/tslibs/timezones.pyx', 'pandas/_libs/tslibs/fields.pyx', 'pandas/_libs/tslibs/frequencies.pyx', @@ -486,6 +487,10 @@ def pxd(name): 'depends': tseries_depends, 'sources': ['pandas/_libs/src/datetime/np_datetime.c', 'pandas/_libs/src/datetime/np_datetime_strings.c']}, + '_libs.tslibs.npy_dtime': {'pyxfile': '_libs/tslibs/npy_dtime', + 'depends': tseries_depends, + 'sources': ['pandas/_libs/src/datetime/np_datetime.c', + 'pandas/_libs/src/datetime/np_datetime_strings.c']}, '_libs.tslibs.timezones': {'pyxfile': '_libs/tslibs/timezones'}, '_libs.tslibs.fields': {'pyxfile': '_libs/tslibs/fields', 'depends': tseries_depends, From 67b3ec3e596fb53132d462adcb66330ed688ee8e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 6 Oct 2017 10:31:14 -0700 Subject: [PATCH 02/13] whitespace fixup --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index ee4ce869cd041..3f87548cb04d7 100755 --- a/setup.py +++ b/setup.py @@ -488,9 +488,10 @@ def pxd(name): 'sources': ['pandas/_libs/src/datetime/np_datetime.c', 'pandas/_libs/src/datetime/np_datetime_strings.c']}, '_libs.tslibs.npy_dtime': {'pyxfile': '_libs/tslibs/npy_dtime', - 'depends': tseries_depends, - 'sources': ['pandas/_libs/src/datetime/np_datetime.c', - 'pandas/_libs/src/datetime/np_datetime_strings.c']}, + 'depends': ['pandas/_libs/src/datetime/np_datetime.h', + 'pandas/_libs/src/datetime/np_datetime_strings.h'], + 'sources': ['pandas/_libs/src/datetime/np_datetime.c', + 'pandas/_libs/src/datetime/np_datetime_strings.c']}, '_libs.tslibs.timezones': {'pyxfile': '_libs/tslibs/timezones'}, '_libs.tslibs.fields': {'pyxfile': '_libs/tslibs/fields', 'depends': tseries_depends, From 76cad9ab9c7909c28ee8370fecc04fce782a4298 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 6 Oct 2017 12:01:47 -0700 Subject: [PATCH 03/13] fixup typo --- pandas/_libs/tslibs/strptime.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index 012c4d1f093b7..01f9be632a5ca 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -321,7 +321,7 @@ def array_strptime(ndarray[object] values, object fmt, iresult[i] = dtstruct_to_dt64(&dts) try: - check_dts_bounds(&dts): + check_dts_bounds(&dts) except ValueError: if is_coerce: iresult[i] = NPY_NAT From ce7197db25da217e63e810c85b3833b9fe0c089d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 6 Oct 2017 19:11:58 -0700 Subject: [PATCH 04/13] flake8 fixup --- pandas/_libs/tslibs/npy_dtime.pyx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/tslibs/npy_dtime.pyx b/pandas/_libs/tslibs/npy_dtime.pyx index e802566d823ad..5aa424036d008 100644 --- a/pandas/_libs/tslibs/npy_dtime.pyx +++ b/pandas/_libs/tslibs/npy_dtime.pyx @@ -28,15 +28,19 @@ cdef extern from "../src/datetime/np_datetime.h": int cmp_pandas_datetimestruct(pandas_datetimestruct *a, pandas_datetimestruct *b) - npy_datetime pandas_datetimestruct_to_datetime(PANDAS_DATETIMEUNIT fr, - pandas_datetimestruct *d) nogil + npy_datetime pandas_datetimestruct_to_datetime( + PANDAS_DATETIMEUNIT fr, + pandas_datetimestruct *d) nogil + void pandas_datetime_to_datetimestruct(npy_datetime val, PANDAS_DATETIMEUNIT fr, pandas_datetimestruct *result) nogil + pandas_datetimestruct _NS_MIN_DTS, _NS_MAX_DTS # ---------------------------------------------------------------------- + class OutOfBoundsDatetime(ValueError): pass @@ -52,7 +56,7 @@ cdef inline check_dts_bounds(pandas_datetimestruct *dts): elif (dts.year >= 2262 and cmp_pandas_datetimestruct(dts, &_NS_MAX_DTS) == 1): error = True - + if error: fmt = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (dts.year, dts.month, dts.day, dts.hour, @@ -70,7 +74,8 @@ cdef inline int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil: return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts) -cdef inline void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil: +cdef inline void dt64_to_dtstruct(int64_t dt64, + pandas_datetimestruct* out) nogil: """Convenience function to call pandas_datetime_to_datetimestruct with the by-far-most-common frequency PANDAS_FR_ns""" pandas_datetime_to_datetimestruct(dt64, PANDAS_FR_ns, out) From a1f8fc162fa0dc1f393f5d8ad5453091ece7941a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 15 Oct 2017 09:01:06 -0700 Subject: [PATCH 05/13] Rename npy_dtime-->np_datetime --- pandas/_libs/tslib.pyx | 4 ++-- pandas/_libs/tslibs/{npy_dtime.pxd => np_datetime.pxd} | 0 pandas/_libs/tslibs/{npy_dtime.pyx => np_datetime.pyx} | 0 pandas/_libs/tslibs/strptime.pyx | 4 ++-- setup.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) rename pandas/_libs/tslibs/{npy_dtime.pxd => np_datetime.pxd} (100%) rename pandas/_libs/tslibs/{npy_dtime.pyx => np_datetime.pyx} (100%) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 69a7af53a96f1..220de49eac75c 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -56,8 +56,8 @@ from datetime cimport ( from datetime import timedelta, datetime from datetime import time as datetime_time -from tslibs.npy_dtime cimport check_dts_bounds as _check_dts_bounds -from tslibs.npy_dtime import OutOfBoundsDatetime +from tslibs.np_datetime cimport check_dts_bounds as _check_dts_bounds +from tslibs.np_datetime import OutOfBoundsDatetime from khash cimport ( khiter_t, diff --git a/pandas/_libs/tslibs/npy_dtime.pxd b/pandas/_libs/tslibs/np_datetime.pxd similarity index 100% rename from pandas/_libs/tslibs/npy_dtime.pxd rename to pandas/_libs/tslibs/np_datetime.pxd diff --git a/pandas/_libs/tslibs/npy_dtime.pyx b/pandas/_libs/tslibs/np_datetime.pyx similarity index 100% rename from pandas/_libs/tslibs/npy_dtime.pyx rename to pandas/_libs/tslibs/np_datetime.pyx diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index 01f9be632a5ca..3121d2cfd534b 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -33,8 +33,8 @@ from numpy cimport ndarray, int64_t from datetime import date as datetime_date from datetime cimport datetime -from npy_dtime cimport (check_dts_bounds, - dtstruct_to_dt64, pandas_datetimestruct) +from np_datetime cimport (check_dts_bounds, + dtstruct_to_dt64, pandas_datetimestruct) from util cimport is_string_object, get_nat diff --git a/setup.py b/setup.py index 3f87548cb04d7..bd05ea4d7db0e 100755 --- a/setup.py +++ b/setup.py @@ -342,7 +342,7 @@ class CheckSDist(sdist_class): 'pandas/_libs/sparse.pyx', 'pandas/_libs/parsers.pyx', 'pandas/_libs/tslibs/strptime.pyx', - 'pandas/_libs/tslibs/npy_dtime.pyx', + 'pandas/_libs/tslibs/np_datetime.pyx', 'pandas/_libs/tslibs/timezones.pyx', 'pandas/_libs/tslibs/fields.pyx', 'pandas/_libs/tslibs/frequencies.pyx', @@ -487,7 +487,7 @@ def pxd(name): 'depends': tseries_depends, 'sources': ['pandas/_libs/src/datetime/np_datetime.c', 'pandas/_libs/src/datetime/np_datetime_strings.c']}, - '_libs.tslibs.npy_dtime': {'pyxfile': '_libs/tslibs/npy_dtime', + '_libs.tslibs.np_datetime': {'pyxfile': '_libs/tslibs/np_datetime', 'depends': ['pandas/_libs/src/datetime/np_datetime.h', 'pandas/_libs/src/datetime/np_datetime_strings.h'], 'sources': ['pandas/_libs/src/datetime/np_datetime.c', From a3458365dab36e2a093a9b4605e04b0ce4c7d273 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 21 Oct 2017 13:05:05 -0700 Subject: [PATCH 06/13] flake8 fixup --- pandas/_libs/tslibs/np_datetime.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index 5aa424036d008..d7b155970b3d6 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -28,9 +28,9 @@ cdef extern from "../src/datetime/np_datetime.h": int cmp_pandas_datetimestruct(pandas_datetimestruct *a, pandas_datetimestruct *b) - npy_datetime pandas_datetimestruct_to_datetime( - PANDAS_DATETIMEUNIT fr, - pandas_datetimestruct *d) nogil + npy_datetime pandas_datetimestruct_to_datetime(PANDAS_DATETIMEUNIT fr, + pandas_datetimestruct *d + ) nogil void pandas_datetime_to_datetimestruct(npy_datetime val, PANDAS_DATETIMEUNIT fr, From c0b399ed1882bbbe7c84c8061586c5c98301322a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 23 Oct 2017 07:59:27 -0700 Subject: [PATCH 07/13] re-privatize _check_dts_bounds --- pandas/_libs/tslib.pyx | 2 +- pandas/_libs/tslibs/np_datetime.pxd | 2 +- pandas/_libs/tslibs/np_datetime.pyx | 2 +- pandas/_libs/tslibs/strptime.pyx | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index b67db4e953b9a..f210964cf679b 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -57,7 +57,7 @@ from datetime cimport ( from datetime import timedelta, datetime from datetime import time as datetime_time -from tslibs.np_datetime cimport check_dts_bounds as _check_dts_bounds +from tslibs.np_datetime cimport _check_dts_bounds from tslibs.np_datetime import OutOfBoundsDatetime from khash cimport ( diff --git a/pandas/_libs/tslibs/np_datetime.pxd b/pandas/_libs/tslibs/np_datetime.pxd index d4079aae68900..f423aa2e549ce 100644 --- a/pandas/_libs/tslibs/np_datetime.pxd +++ b/pandas/_libs/tslibs/np_datetime.pxd @@ -10,7 +10,7 @@ cdef extern from "../src/datetime/np_datetime.h": int32_t month, day, hour, min, sec, us, ps, as -cdef check_dts_bounds(pandas_datetimestruct *dts) +cdef _check_dts_bounds(pandas_datetimestruct *dts) cdef int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil cdef void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index d7b155970b3d6..b4373c4d17ac8 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -45,7 +45,7 @@ class OutOfBoundsDatetime(ValueError): pass -cdef inline check_dts_bounds(pandas_datetimestruct *dts): +cdef inline _check_dts_bounds(pandas_datetimestruct *dts): """Returns True if an error needs to be raised""" cdef: bint error = False diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index ecb0eb0e51e1d..13294349626ad 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -33,7 +33,7 @@ from numpy cimport ndarray, int64_t from datetime import date as datetime_date from datetime cimport datetime -from np_datetime cimport (check_dts_bounds, +from np_datetime cimport (_check_dts_bounds, dtstruct_to_dt64, pandas_datetimestruct) from util cimport is_string_object, get_nat @@ -331,7 +331,7 @@ def array_strptime(ndarray[object] values, object fmt, iresult[i] = dtstruct_to_dt64(&dts) try: - check_dts_bounds(&dts) + _check_dts_bounds(&dts) except ValueError: if is_coerce: iresult[i] = NPY_NAT From 179138925dccd9ff670f0c089e3a58c62a03ae47 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 23 Oct 2017 13:05:35 -0700 Subject: [PATCH 08/13] dummy commit to force ci re-run --- pandas/_libs/tslibs/np_datetime.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index b4373c4d17ac8..3d6e388add7dc 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -7,7 +7,6 @@ cdef extern from "numpy/ndarrayobject.h": ctypedef int64_t npy_timedelta ctypedef int64_t npy_datetime - cdef extern from "../src/datetime/np_datetime.h": ctypedef enum PANDAS_DATETIMEUNIT: PANDAS_FR_Y From 16603d4f394b7e52b803a702d2a5a6ec6502211a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 23 Oct 2017 16:34:19 -0700 Subject: [PATCH 09/13] re-de-privatize --- pandas/_libs/tslib.pyx | 2 +- pandas/_libs/tslibs/np_datetime.pxd | 2 +- pandas/_libs/tslibs/np_datetime.pyx | 2 +- pandas/_libs/tslibs/strptime.pyx | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index f210964cf679b..b67db4e953b9a 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -57,7 +57,7 @@ from datetime cimport ( from datetime import timedelta, datetime from datetime import time as datetime_time -from tslibs.np_datetime cimport _check_dts_bounds +from tslibs.np_datetime cimport check_dts_bounds as _check_dts_bounds from tslibs.np_datetime import OutOfBoundsDatetime from khash cimport ( diff --git a/pandas/_libs/tslibs/np_datetime.pxd b/pandas/_libs/tslibs/np_datetime.pxd index f423aa2e549ce..d4079aae68900 100644 --- a/pandas/_libs/tslibs/np_datetime.pxd +++ b/pandas/_libs/tslibs/np_datetime.pxd @@ -10,7 +10,7 @@ cdef extern from "../src/datetime/np_datetime.h": int32_t month, day, hour, min, sec, us, ps, as -cdef _check_dts_bounds(pandas_datetimestruct *dts) +cdef check_dts_bounds(pandas_datetimestruct *dts) cdef int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil cdef void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index 3d6e388add7dc..8a0b71251107d 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -44,7 +44,7 @@ class OutOfBoundsDatetime(ValueError): pass -cdef inline _check_dts_bounds(pandas_datetimestruct *dts): +cdef inline check_dts_bounds(pandas_datetimestruct *dts): """Returns True if an error needs to be raised""" cdef: bint error = False diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index 13294349626ad..ecb0eb0e51e1d 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -33,7 +33,7 @@ from numpy cimport ndarray, int64_t from datetime import date as datetime_date from datetime cimport datetime -from np_datetime cimport (_check_dts_bounds, +from np_datetime cimport (check_dts_bounds, dtstruct_to_dt64, pandas_datetimestruct) from util cimport is_string_object, get_nat @@ -331,7 +331,7 @@ def array_strptime(ndarray[object] values, object fmt, iresult[i] = dtstruct_to_dt64(&dts) try: - _check_dts_bounds(&dts) + check_dts_bounds(&dts) except ValueError: if is_coerce: iresult[i] = NPY_NAT From c05cec7d87f1bc40079da6f0b04e0dafc6720b07 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 09:47:20 -0700 Subject: [PATCH 10/13] edits per reviewer suggestions --- pandas/_libs/tslib.pyx | 32 ++++++++++++++--------------- pandas/_libs/tslibs/np_datetime.pxd | 2 +- pandas/_libs/tslibs/np_datetime.pyx | 2 +- pandas/_libs/tslibs/strptime.pyx | 3 +-- setup.py | 27 ++++++++++++------------ 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index e3e61e3263e6f..33542b5a82853 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -58,7 +58,7 @@ from datetime cimport ( from datetime import timedelta, datetime from datetime import time as datetime_time -from tslibs.np_datetime cimport check_dts_bounds as _check_dts_bounds +from tslibs.np_datetime cimport check_dts_bounds from tslibs.np_datetime import OutOfBoundsDatetime from khash cimport ( @@ -735,7 +735,7 @@ class Timestamp(_Timestamp): ts = convert_datetime_to_tsobject(ts_input, _tzinfo) value = ts.value + (dts.ps // 1000) if value != NPY_NAT: - _check_dts_bounds(&dts) + check_dts_bounds(&dts) return create_timestamp_from_ts(value, dts, _tzinfo, self.freq) @@ -1603,7 +1603,7 @@ cdef convert_to_tsobject(object ts, object tz, object unit, 'Timestamp'.format(ts, type(ts))) if obj.value != NPY_NAT: - _check_dts_bounds(&obj.dts) + check_dts_bounds(&obj.dts) if tz is not None: _localize_tso(obj, tz) @@ -1684,7 +1684,7 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz, obj.value += nanos obj.dts.ps = nanos * 1000 - _check_dts_bounds(&obj.dts) + check_dts_bounds(&obj.dts) return obj @@ -1720,12 +1720,12 @@ cpdef convert_str_to_tsobject(object ts, object tz, object unit, _string_to_dts(ts, &obj.dts, &out_local, &out_tzoffset) obj.value = pandas_datetimestruct_to_datetime( PANDAS_FR_ns, &obj.dts) - _check_dts_bounds(&obj.dts) + check_dts_bounds(&obj.dts) if out_local == 1: obj.tzinfo = pytz.FixedOffset(out_tzoffset) obj.value = tz_convert_single(obj.value, obj.tzinfo, 'UTC') if tz is None: - _check_dts_bounds(&obj.dts) + check_dts_bounds(&obj.dts) return obj else: # Keep the converter same as PyDateTime's @@ -1768,7 +1768,7 @@ def _test_parse_iso8601(object ts): _string_to_dts(ts, &obj.dts, &out_local, &out_tzoffset) obj.value = pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &obj.dts) - _check_dts_bounds(&obj.dts) + check_dts_bounds(&obj.dts) if out_local == 1: obj.tzinfo = pytz.FixedOffset(out_tzoffset) obj.value = tz_convert_single(obj.value, obj.tzinfo, 'UTC') @@ -1879,13 +1879,13 @@ def datetime_to_datetime64(ndarray[object] values): _ts = convert_datetime_to_tsobject(val, None) iresult[i] = _ts.value - _check_dts_bounds(&_ts.dts) + check_dts_bounds(&_ts.dts) else: if inferred_tz is not None: raise ValueError('Cannot mix tz-aware with ' 'tz-naive values') iresult[i] = _pydatetime_to_dts(val, &dts) - _check_dts_bounds(&dts) + check_dts_bounds(&dts) else: raise TypeError('Unrecognized value type: %s' % type(val)) @@ -2198,7 +2198,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', _ts = convert_datetime_to_tsobject(val, None) iresult[i] = _ts.value try: - _check_dts_bounds(&_ts.dts) + check_dts_bounds(&_ts.dts) except ValueError: if is_coerce: iresult[i] = NPY_NAT @@ -2213,7 +2213,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', if is_timestamp(val): iresult[i] += val.nanosecond try: - _check_dts_bounds(&dts) + check_dts_bounds(&dts) except ValueError: if is_coerce: iresult[i] = NPY_NAT @@ -2223,7 +2223,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', elif PyDate_Check(val): iresult[i] = _date_to_datetime64(val, &dts) try: - _check_dts_bounds(&dts) + check_dts_bounds(&dts) seen_datetime = 1 except ValueError: if is_coerce: @@ -2280,7 +2280,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', tz = pytz.FixedOffset(out_tzoffset) value = tz_convert_single(value, tz, 'UTC') iresult[i] = value - _check_dts_bounds(&dts) + check_dts_bounds(&dts) except ValueError: # if requiring iso8601 strings, skip trying other formats if require_iso8601: @@ -2379,7 +2379,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, yearfirst=yearfirst) _pydatetime_to_dts(oresult[i], &dts) - _check_dts_bounds(&dts) + check_dts_bounds(&dts) except Exception: if is_raise: raise @@ -3185,7 +3185,7 @@ cdef inline _get_datetime64_nanos(object val): if unit != PANDAS_FR_ns: pandas_datetime_to_datetimestruct(ival, unit, &dts) - _check_dts_bounds(&dts) + check_dts_bounds(&dts) return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts) else: return ival @@ -3213,7 +3213,7 @@ def cast_to_nanoseconds(ndarray arr): if ivalues[i] != NPY_NAT: pandas_datetime_to_datetimestruct(ivalues[i], unit, &dts) iresult[i] = pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts) - _check_dts_bounds(&dts) + check_dts_bounds(&dts) else: iresult[i] = NPY_NAT diff --git a/pandas/_libs/tslibs/np_datetime.pxd b/pandas/_libs/tslibs/np_datetime.pxd index d4079aae68900..c60e9c9414333 100644 --- a/pandas/_libs/tslibs/np_datetime.pxd +++ b/pandas/_libs/tslibs/np_datetime.pxd @@ -10,7 +10,7 @@ cdef extern from "../src/datetime/np_datetime.h": int32_t month, day, hour, min, sec, us, ps, as -cdef check_dts_bounds(pandas_datetimestruct *dts) +cdef void check_dts_bounds(pandas_datetimestruct *dts) cdef int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil cdef void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index 8a0b71251107d..c53ca6f2165d5 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -44,7 +44,7 @@ class OutOfBoundsDatetime(ValueError): pass -cdef inline check_dts_bounds(pandas_datetimestruct *dts): +cdef inline void check_dts_bounds(pandas_datetimestruct *dts): """Returns True if an error needs to be raised""" cdef: bint error = False diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index ecb0eb0e51e1d..4a141c7b56428 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -336,8 +336,7 @@ def array_strptime(ndarray[object] values, object fmt, if is_coerce: iresult[i] = NPY_NAT continue - else: - raise + raise return result diff --git a/setup.py b/setup.py index 79fc26bdcaef5..b2d8a54748b5d 100755 --- a/setup.py +++ b/setup.py @@ -469,12 +469,11 @@ def pxd(name): 'pandas/_libs/src/parse_helper.h', 'pandas/_libs/src/compat_helper.h'] - -tseries_depends = ['pandas/_libs/src/datetime/np_datetime.h', - 'pandas/_libs/src/datetime/np_datetime_strings.h', - 'pandas/_libs/src/datetime.pxd'] -npdt_srces = ['pandas/_libs/src/datetime/np_datetime.c', - 'pandas/_libs/src/datetime/np_datetime_strings.c'] +np_datetime_headers = ['pandas/_libs/src/datetime/np_datetime.h', + 'pandas/_libs/src/datetime/np_datetime_strings.h',] +np_datetime_sources = ['pandas/_libs/src/datetime/np_datetime.c', + 'pandas/_libs/src/datetime/np_datetime_strings.c'] +tseries_depends = np_datetime_headers + ['pandas/_libs/src/datetime.pxd'] # some linux distros require it libraries = ['m'] if not is_platform_windows() else [] @@ -489,30 +488,30 @@ def pxd(name): _pxi_dep['hashtable'])}, '_libs.tslibs.strptime': {'pyxfile': '_libs/tslibs/strptime', 'depends': tseries_depends, - 'sources': npdt_srces}, + 'sources': np_datetime_sources}, '_libs.tslib': {'pyxfile': '_libs/tslib', 'pxdfiles': ['_libs/src/util', '_libs/lib'], 'depends': tseries_depends, - 'sources': npdt_srces}, + 'sources': np_datetime_sources}, '_libs.tslibs.np_datetime': {'pyxfile': '_libs/tslibs/np_datetime', - 'depends': tseries_depends[:2], - 'sources': npdt_srces}, + 'depends': np_datetime_headers, + 'sources': np_datetime_sources}, '_libs.tslibs.timedeltas': {'pyxfile': '_libs/tslibs/timedeltas'}, '_libs.tslibs.timezones': {'pyxfile': '_libs/tslibs/timezones'}, '_libs.tslibs.fields': {'pyxfile': '_libs/tslibs/fields', 'depends': tseries_depends, - 'sources': npdt_srces}, + 'sources': np_datetime_sources}, '_libs.period': {'pyxfile': '_libs/period', 'depends': (tseries_depends + ['pandas/_libs/src/period_helper.h']), - 'sources': npdt_srces + [ + 'sources': np_datetime_sources + [ 'pandas/_libs/src/period_helper.c']}, '_libs.tslibs.parsing': {'pyxfile': '_libs/tslibs/parsing', 'pxdfiles': ['_libs/src/util']}, '_libs.tslibs.frequencies': {'pyxfile': '_libs/tslibs/frequencies', 'pxdfiles': ['_libs/src/util']}, '_libs.index': {'pyxfile': '_libs/index', - 'sources': npdt_srces, + 'sources': np_datetime_sources, 'pxdfiles': ['_libs/src/util', '_libs/hashtable'], 'depends': _pxi_dep['index']}, '_libs.algos': {'pyxfile': '_libs/algos', @@ -625,7 +624,7 @@ def pxd(name): 'pandas/_libs/src/ujson/python/JSONtoObj.c', 'pandas/_libs/src/ujson/lib/ultrajsonenc.c', 'pandas/_libs/src/ujson/lib/ultrajsondec.c'] + - npdt_srces), + np_datetime_sources), include_dirs=(['pandas/_libs/src/ujson/python', 'pandas/_libs/src/ujson/lib', 'pandas/_libs/src/datetime'] + From c0b821670f56f1298d66a533beee6e9e4ce69d1c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 11:49:36 -0700 Subject: [PATCH 11/13] revert void; format per reviewer suggestion --- pandas/_libs/tslibs/np_datetime.pxd | 2 +- pandas/_libs/tslibs/np_datetime.pyx | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/tslibs/np_datetime.pxd b/pandas/_libs/tslibs/np_datetime.pxd index c60e9c9414333..d4079aae68900 100644 --- a/pandas/_libs/tslibs/np_datetime.pxd +++ b/pandas/_libs/tslibs/np_datetime.pxd @@ -10,7 +10,7 @@ cdef extern from "../src/datetime/np_datetime.h": int32_t month, day, hour, min, sec, us, ps, as -cdef void check_dts_bounds(pandas_datetimestruct *dts) +cdef check_dts_bounds(pandas_datetimestruct *dts) cdef int64_t dtstruct_to_dt64(pandas_datetimestruct* dts) nogil cdef void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index c53ca6f2165d5..38158c816812d 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -44,8 +44,9 @@ class OutOfBoundsDatetime(ValueError): pass -cdef inline void check_dts_bounds(pandas_datetimestruct *dts): - """Returns True if an error needs to be raised""" +cdef inline check_dts_bounds(pandas_datetimestruct *dts): + """Raises OutOfBoundsDatetime if the given date is outside the range that + can be represented by nanosecond-resolution 64-bit integers.""" cdef: bint error = False @@ -61,7 +62,7 @@ cdef inline void check_dts_bounds(pandas_datetimestruct *dts): dts.day, dts.hour, dts.min, dts.sec) raise OutOfBoundsDatetime( - 'Out of bounds nanosecond timestamp: %s' % fmt) + 'Out of bounds nanosecond timestamp: {fmt}'.format(fmt=fmt)) # ---------------------------------------------------------------------- From 7ac961576c5bcc6c50f82ff43a8bd0573b9d4558 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 22:10:40 -0700 Subject: [PATCH 12/13] flake8 typo fixup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfc22757b330f..ed9a2e196c006 100755 --- a/setup.py +++ b/setup.py @@ -471,7 +471,7 @@ def pxd(name): 'pandas/_libs/src/compat_helper.h'] np_datetime_headers = ['pandas/_libs/src/datetime/np_datetime.h', - 'pandas/_libs/src/datetime/np_datetime_strings.h',] + 'pandas/_libs/src/datetime/np_datetime_strings.h'] np_datetime_sources = ['pandas/_libs/src/datetime/np_datetime.c', 'pandas/_libs/src/datetime/np_datetime_strings.c'] tseries_depends = np_datetime_headers + ['pandas/_libs/src/datetime.pxd'] From e017a52984831c0312344a20b1ad31f706612b80 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 13:27:39 -0700 Subject: [PATCH 13/13] whitespace fixup --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ed9a2e196c006..2843ab6587412 100755 --- a/setup.py +++ b/setup.py @@ -496,8 +496,8 @@ def pxd(name): 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.np_datetime': {'pyxfile': '_libs/tslibs/np_datetime', - 'depends': np_datetime_headers, - 'sources': np_datetime_sources}, + 'depends': np_datetime_headers, + 'sources': np_datetime_sources}, '_libs.tslibs.timedeltas': {'pyxfile': '_libs/tslibs/timedeltas'}, '_libs.tslibs.timezones': {'pyxfile': '_libs/tslibs/timezones'}, '_libs.tslibs.fields': {'pyxfile': '_libs/tslibs/fields',