Implement numpy_helper functions directly in cython (#18059)

Author: jbrockmendel

pandas/_libs/lib.pyx

@@ -48,9 +48,8 @@ from cpython.datetime cimport (PyDateTime_Check, PyDate_Check,
                                PyTime_Check, PyDelta_Check,
                                PyDateTime_IMPORT)
 PyDateTime_IMPORT
-# this is our tseries.pxd
-from datetime cimport get_timedelta64_value, get_datetime64_value
 
+from tslibs.np_datetime cimport get_timedelta64_value, get_datetime64_value
 
 from tslib cimport _check_all_nulls
 import tslib

pandas/_libs/src/datetime.pxd

@@ -1,52 +1,20 @@
 # cython: profile=False
-from numpy cimport int64_t, int32_t, npy_int64, npy_int32, ndarray
-from cpython cimport PyObject
+from numpy cimport int64_t, npy_int64, npy_int32
 
 from cpython cimport PyUnicode_Check, PyUnicode_AsASCIIString
 
 
-cdef extern from "datetime.h":
-
-    ctypedef class datetime.date [object PyDateTime_Date]:
-        pass
-
-    ctypedef class datetime.datetime [object PyDateTime_DateTime]:
-        pass
-
-    ctypedef class datetime.timedelta [object PyDateTime_Delta]:
-        pass
-
-    void PyDateTime_IMPORT()
-
-    int PyDateTime_GET_YEAR(date)
-    int PyDateTime_GET_MONTH(date)
-    int PyDateTime_GET_DAY(date)
-    int PyDateTime_DATE_GET_HOUR(object o)
-    int PyDateTime_DATE_GET_MINUTE(object o)
-    int PyDateTime_DATE_GET_SECOND(object o)
-    int PyDateTime_DATE_GET_MICROSECOND(object o)
-    int PyDateTime_TIME_GET_HOUR(object o)
-    int PyDateTime_TIME_GET_MINUTE(object o)
-    int PyDateTime_TIME_GET_SECOND(object o)
-    int PyDateTime_TIME_GET_MICROSECOND(object o)
-    bint PyDateTime_Check(object o)
-    bint PyDate_Check(object o)
-    bint PyTime_Check(object o)
-    bint PyDelta_Check(object o)
-    object PyDateTime_FromDateAndTime(int year, int month, int day, int hour,
-                                      int minute, int second, int us)
-
 cdef extern from "numpy/ndarrayobject.h":
 
     ctypedef int64_t npy_timedelta
     ctypedef int64_t npy_datetime
 
     ctypedef enum NPY_CASTING:
-            NPY_NO_CASTING
-            NPY_EQUIV_CASTING
-            NPY_SAFE_CASTING
-            NPY_SAME_KIND_CASTING
-            NPY_UNSAFE_CASTING
+        NPY_NO_CASTING
+        NPY_EQUIV_CASTING
+        NPY_SAFE_CASTING
+        NPY_SAME_KIND_CASTING
+        NPY_UNSAFE_CASTING
 
 
 cdef extern from "numpy_helper.h":
@@ -79,9 +47,6 @@ cdef extern from "datetime/np_datetime.h":
         npy_int64 year
         npy_int32 month, day, hour, min, sec, us, ps, as
 
-    int cmp_pandas_datetimestruct(pandas_datetimestruct *a,
-                                  pandas_datetimestruct *b)
-
     npy_datetime pandas_datetimestruct_to_datetime(PANDAS_DATETIMEUNIT fr,
                                                    pandas_datetimestruct *d) nogil
     void pandas_datetime_to_datetimestruct(npy_datetime val,
@@ -102,8 +67,6 @@ cdef extern from "datetime/np_datetime_strings.h":
                                 PANDAS_DATETIMEUNIT *out_bestunit,
                                 npy_bool *out_special)
 
-    # int parse_python_string(object obj, pandas_datetimestruct *out) except -1
-
 
 
 
@@ -134,17 +97,3 @@ cdef inline int _cstring_to_dts(char *val, int length,
                                      NPY_UNSAFE_CASTING,
                                      dts, out_local, out_tzoffset, &out_bestunit, &special)
     return result
-
-
-cdef inline bint check_dts_bounds(pandas_datetimestruct *dts):
-    """Returns True if an error needs to be raised"""
-    cdef:
-        bint error = False
-
-    if (dts.year <= 1677 and
-            cmp_pandas_datetimestruct(dts, &_NS_MIN_DTS) == -1):
-        error = True
-    elif (dts.year >= 2262 and
-          cmp_pandas_datetimestruct(dts, &_NS_MAX_DTS) == 1):
-        error = True
-    return error

pandas/_libs/tslib.pyx

@@ -41,12 +41,8 @@ PyDateTime_IMPORT
 from datetime cimport (
     pandas_datetime_to_datetimestruct,
     days_per_month_table,
-    get_datetime64_value,
-    get_timedelta64_value,
-    get_datetime64_unit,
     PANDAS_DATETIMEUNIT,
     _string_to_dts,
-    npy_datetime,
     is_leapyear,
     dayofweek,
     PANDAS_FR_ns)
@@ -59,7 +55,10 @@ from tslibs.np_datetime cimport (check_dts_bounds,
                                  cmp_scalar,
                                  pandas_datetimestruct,
                                  dt64_to_dtstruct, dtstruct_to_dt64,
-                                 pydatetime_to_dt64, pydate_to_dt64)
+                                 pydatetime_to_dt64, pydate_to_dt64,
+                                 npy_datetime,
+                                 get_datetime64_unit, get_datetime64_value,
+                                 get_timedelta64_value)
 from tslibs.np_datetime import OutOfBoundsDatetime
 
 from khash cimport (

pandas/_libs/tslibs/np_datetime.pxd

@@ -5,12 +5,46 @@ from cpython.datetime cimport date, datetime
 
 from numpy cimport int64_t, int32_t
 
+cdef extern from "numpy/ndarrayobject.h":
+    ctypedef int64_t npy_timedelta
+    ctypedef int64_t npy_datetime
+
+cdef extern from "numpy/ndarraytypes.h":
+    ctypedef struct PyArray_DatetimeMetaData:
+        PANDAS_DATETIMEUNIT base
+        int64_t num
+
+cdef extern from "numpy/arrayscalars.h":
+    ctypedef struct PyDatetimeScalarObject:
+        # PyObject_HEAD
+        npy_datetime obval
+        PyArray_DatetimeMetaData obmeta
+
+    ctypedef struct PyTimedeltaScalarObject:
+        # PyObject_HEAD
+        npy_timedelta obval
+        PyArray_DatetimeMetaData obmeta
 
 cdef extern from "../src/datetime/np_datetime.h":
     ctypedef struct pandas_datetimestruct:
         int64_t year
         int32_t month, day, hour, min, sec, us, ps, as
 
+    ctypedef enum PANDAS_DATETIMEUNIT:
+        PANDAS_FR_Y
+        PANDAS_FR_M
+        PANDAS_FR_W
+        PANDAS_FR_D
+        PANDAS_FR_B
+        PANDAS_FR_h
+        PANDAS_FR_m
+        PANDAS_FR_s
+        PANDAS_FR_ms
+        PANDAS_FR_us
+        PANDAS_FR_ns
+        PANDAS_FR_ps
+        PANDAS_FR_fs
+        PANDAS_FR_as
 
 cdef int reverse_ops[6]
 
@@ -23,3 +57,7 @@ cdef void dt64_to_dtstruct(int64_t dt64, pandas_datetimestruct* out) nogil
 
 cdef int64_t pydatetime_to_dt64(datetime val, pandas_datetimestruct *dts)
 cdef int64_t pydate_to_dt64(date val, pandas_datetimestruct *dts)
+
+cdef npy_datetime get_datetime64_value(object obj) nogil
+cdef npy_timedelta get_timedelta64_value(object obj) nogil
+cdef PANDAS_DATETIMEUNIT get_datetime64_unit(object obj) nogil

pandas/_libs/tslibs/np_datetime.pyx

@@ -14,27 +14,7 @@ PyDateTime_IMPORT
 
 from numpy cimport int64_t
 
-cdef extern from "numpy/ndarrayobject.h":
-    ctypedef int64_t npy_timedelta
-    ctypedef int64_t npy_datetime
-
 cdef extern from "../src/datetime/np_datetime.h":
-    ctypedef enum PANDAS_DATETIMEUNIT:
-        PANDAS_FR_Y
-        PANDAS_FR_M
-        PANDAS_FR_W
-        PANDAS_FR_D
-        PANDAS_FR_B
-        PANDAS_FR_h
-        PANDAS_FR_m
-        PANDAS_FR_s
-        PANDAS_FR_ms
-        PANDAS_FR_us
-        PANDAS_FR_ns
-        PANDAS_FR_ps
-        PANDAS_FR_fs
-        PANDAS_FR_as
-
     int cmp_pandas_datetimestruct(pandas_datetimestruct *a,
                                   pandas_datetimestruct *b)
 
@@ -48,6 +28,32 @@ cdef extern from "../src/datetime/np_datetime.h":
 
     pandas_datetimestruct _NS_MIN_DTS, _NS_MAX_DTS
 
+# ----------------------------------------------------------------------
+# numpy object inspection
+
+cdef inline npy_datetime get_datetime64_value(object obj) nogil:
+    """
+    returns the int64 value underlying scalar numpy datetime64 object
+
+    Note that to interpret this as a datetime, the corresponding unit is
+    also needed.  That can be found using `get_datetime64_unit`.
+    """
+    return (<PyDatetimeScalarObject*>obj).obval
+
+
+cdef inline npy_timedelta get_timedelta64_value(object obj) nogil:
+    """
+    returns the int64 value underlying scalar numpy timedelta64 object
+    """
+    return (<PyTimedeltaScalarObject*>obj).obval
+
+
+cdef inline PANDAS_DATETIMEUNIT get_datetime64_unit(object obj) nogil:
+    """
+    returns the unit part of the dtype for a numpy datetime64 object.
+    """
+    return <PANDAS_DATETIMEUNIT>(<PyDatetimeScalarObject*>obj).obmeta.base
+
 # ----------------------------------------------------------------------
 # Comparison