From f40c73fc32fd8c7b87367269929c03a93705d617 Mon Sep 17 00:00:00 2001 From: Manraj Singh Grover Date: Tue, 7 Nov 2017 21:49:32 +0530 Subject: [PATCH 1/6] Fix 18121: Add .pxd linting to lint.sh --- ci/lint.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci/lint.sh b/ci/lint.sh index 43d6ea0c118b0..bf18d175108b4 100755 --- a/ci/lint.sh +++ b/ci/lint.sh @@ -42,6 +42,18 @@ if [ "$LINT" ]; then done echo "Linting *.pxi.in DONE" + echo "Linting *.pxd" + for path in 'src' + do + echo "linting -> pandas/$path" + flake8 pandas/$path --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 + if [ $? -ne "0" ]; then + RET=1 + fi + + done + echo "Linting *.pxd DONE" + # readability/casting: Warnings about C casting instead of C++ casting # runtime/int: Warnings about using C number types instead of C++ ones # build/include_subdir: Warnings about prefacing included header files with directory From 972255ab975594c486b28bc5fa66322239e37f74 Mon Sep 17 00:00:00 2001 From: Manraj Singh Grover Date: Wed, 8 Nov 2017 15:58:38 +0530 Subject: [PATCH 2/6] Recursively search _libs --- ci/lint.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/lint.sh b/ci/lint.sh index bf18d175108b4..9f98b524db197 100755 --- a/ci/lint.sh +++ b/ci/lint.sh @@ -38,19 +38,17 @@ if [ "$LINT" ]; then if [ $? -ne "0" ]; then RET=1 fi - done echo "Linting *.pxi.in DONE" echo "Linting *.pxd" - for path in 'src' + for path in '_libs' do echo "linting -> pandas/$path" flake8 pandas/$path --filename=*.pxd --select=E501,E302,E203,E111,E114,E221,E303,E231,E126,F403 if [ $? -ne "0" ]; then RET=1 fi - done echo "Linting *.pxd DONE" From d6a7e5d6ac30b18bab39922a0be4b5d0086288f5 Mon Sep 17 00:00:00 2001 From: Manraj Singh Grover Date: Wed, 8 Nov 2017 16:13:59 +0530 Subject: [PATCH 3/6] Fix linting issues in *.pxd files --- ci/lint.sh | 2 +- pandas/_libs/src/datetime.pxd | 12 ++++--- pandas/_libs/src/numpy.pxd | 66 +++++++++++++++++++++-------------- pandas/_libs/src/util.pxd | 3 +- 4 files changed, 49 insertions(+), 34 deletions(-) diff --git a/ci/lint.sh b/ci/lint.sh index 9f98b524db197..c26b5f00d5a48 100755 --- a/ci/lint.sh +++ b/ci/lint.sh @@ -9,7 +9,7 @@ RET=0 if [ "$LINT" ]; then # pandas/_libs/src is C code, so no need to search there. - echo "Linting *.py" + echo "Linting *.py" flake8 pandas --filename=*.py --exclude pandas/_libs/src if [ $? -ne "0" ]; then RET=1 diff --git a/pandas/_libs/src/datetime.pxd b/pandas/_libs/src/datetime.pxd index a5ba610dc89dc..3fc3625a06634 100644 --- a/pandas/_libs/src/datetime.pxd +++ b/pandas/_libs/src/datetime.pxd @@ -44,8 +44,9 @@ cdef extern from "datetime/np_datetime.h": npy_int64 year npy_int32 month, day, hour, min, sec, us, ps, as - npy_datetime pandas_datetimestruct_to_datetime(PANDAS_DATETIMEUNIT fr, - pandas_datetimestruct *d) nogil + npy_datetime pandas_datetimestruct_to_datetime( + PANDAS_DATETIMEUNIT fr, pandas_datetimestruct *d) nogil + void pandas_datetime_to_datetimestruct(npy_datetime val, PANDAS_DATETIMEUNIT fr, pandas_datetimestruct *result) nogil @@ -58,12 +59,12 @@ cdef extern from "datetime/np_datetime.h": cdef extern from "datetime/np_datetime_strings.h": int parse_iso_8601_datetime(char *str, int len, PANDAS_DATETIMEUNIT unit, - NPY_CASTING casting, pandas_datetimestruct *out, + NPY_CASTING casting, + pandas_datetimestruct *out, int *out_local, int *out_tzoffset, PANDAS_DATETIMEUNIT *out_bestunit, npy_bool *out_special) - cdef inline int _string_to_dts(object val, pandas_datetimestruct* dts, int* out_local, int* out_tzoffset) except? -1: cdef int result @@ -89,5 +90,6 @@ cdef inline int _cstring_to_dts(char *val, int length, result = parse_iso_8601_datetime(val, length, PANDAS_FR_ns, NPY_UNSAFE_CASTING, - dts, out_local, out_tzoffset, &out_bestunit, &special) + dts, out_local, out_tzoffset, + &out_bestunit, &special) return result diff --git a/pandas/_libs/src/numpy.pxd b/pandas/_libs/src/numpy.pxd index 9ab3b9b1b81ae..b7970ed6ae373 100644 --- a/pandas/_libs/src/numpy.pxd +++ b/pandas/_libs/src/numpy.pxd @@ -152,7 +152,8 @@ cdef extern from "numpy/arrayobject.h": npy_intp NPY_MAX_ELSIZE - ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *, void *) + ctypedef void (*PyArray_VectorUnaryFunc)( + void *, void *, npy_intp, void *, void *) ctypedef class numpy.dtype [object PyArray_Descr]: # Use PyDataType_* macros when possible, however there are no macros @@ -225,7 +226,9 @@ cdef extern from "numpy/arrayobject.h": if copy_shape: # Allocate new buffer for strides and shape info. # This is allocated as one block, strides first. - info.strides = stdlib.malloc(sizeof(Py_ssize_t) * ndim * 2) + info.strides = stdlib.malloc( + sizeof(Py_ssize_t) * ndim * 2) + info.shape = info.strides + ndim for i in range(ndim): info.strides[i] = PyArray_STRIDES(self)[i] @@ -275,7 +278,8 @@ cdef extern from "numpy/arrayobject.h": elif t == NPY_CLONGDOUBLE: f = "Zg" elif t == NPY_OBJECT: f = "O" else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) + raise ValueError( + u"unknown dtype code in numpy.pxd (%d)" % t) info.format = f return else: @@ -294,7 +298,6 @@ cdef extern from "numpy/arrayobject.h": stdlib.free(info.strides) # info.shape was stored after info.strides in the same block - ctypedef signed char npy_bool ctypedef signed char npy_byte @@ -462,7 +465,6 @@ cdef extern from "numpy/arrayobject.h": bint PyArray_ISBEHAVED(ndarray) bint PyArray_ISBEHAVED_RO(ndarray) - bint PyDataType_ISNOTSWAPPED(dtype) bint PyDataType_ISBYTESWAPPED(dtype) @@ -475,7 +477,6 @@ cdef extern from "numpy/arrayobject.h": # bint PyArray_HasArrayInterfaceType(object, dtype, object, object&) # bint PyArray_HasArrayInterface(op, out) - bint PyArray_IsZeroDim(object) # Cannot be supported due to ## ## in macro: # bint PyArray_IsScalar(object, verbatim work) @@ -502,24 +503,28 @@ cdef extern from "numpy/arrayobject.h": unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2) bint PyArray_EquivByteorders(int b1, int b2) object PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) - object PyArray_SimpleNewFromData(int nd, npy_intp* dims, int typenum, void* data) + object PyArray_SimpleNewFromData(int nd, npy_intp* dims, + int typenum, void* data) #object PyArray_SimpleNewFromDescr(int nd, npy_intp* dims, dtype descr) object PyArray_ToScalar(void* data, ndarray arr) void* PyArray_GETPTR1(ndarray m, npy_intp i) void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) - void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) + void* PyArray_GETPTR4(ndarray m, npy_intp i, + npy_intp j, npy_intp k, npy_intp l) void PyArray_XDECREF_ERR(ndarray) # Cannot be supported due to out arg # void PyArray_DESCR_REPLACE(descr) - object PyArray_Copy(ndarray) - object PyArray_FromObject(object op, int type, int min_depth, int max_depth) - object PyArray_ContiguousFromObject(object op, int type, int min_depth, int max_depth) - object PyArray_CopyFromObject(object op, int type, int min_depth, int max_depth) + object PyArray_FromObject(object op, int type, + int min_depth, int max_depth) + object PyArray_ContiguousFromObject(object op, int type, + int min_depth, int max_depth) + object PyArray_CopyFromObject(object op, int type, + int min_depth, int max_depth) object PyArray_Cast(ndarray mp, int type_num) object PyArray_Take(ndarray ap, object items, int axis) @@ -598,8 +603,10 @@ cdef extern from "numpy/arrayobject.h": object PyArray_Dumps (object, int) int PyArray_ValidType (int) void PyArray_UpdateFlags (ndarray, int) - object PyArray_New (type, int, npy_intp *, int, npy_intp *, void *, int, int, object) - #object PyArray_NewFromDescr (type, dtype, int, npy_intp *, npy_intp *, void *, int, object) + object PyArray_New (type, int, npy_intp *, int, npy_intp *, + void *, int, int, object) + #object PyArray_NewFromDescr ( + type, dtype, int, npy_intp *, npy_intp *, void *, int, object) #dtype PyArray_DescrNew (dtype) dtype PyArray_DescrNewFromType (int) double PyArray_GetPriority (object, double) @@ -611,7 +618,8 @@ cdef extern from "numpy/arrayobject.h": int PyArray_Broadcast (broadcast) void PyArray_FillObjectArray (ndarray, object) int PyArray_FillWithScalar (ndarray, object) - npy_bool PyArray_CheckStrides (int, int, npy_intp, npy_intp, npy_intp *, npy_intp *) + npy_bool PyArray_CheckStrides ( + int, int, npy_intp, npy_intp, npy_intp *, npy_intp *) dtype PyArray_DescrNewByteorder (dtype, char) object PyArray_IterAllButAxis (object, int *) #object PyArray_CheckFromAny (object, dtype, int, int, int, object) @@ -782,9 +790,11 @@ cdef inline object PyArray_MultiIterNew4(a, b, c, d): return PyArray_MultiIterNew(4, a, b, c, d) cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): - return PyArray_MultiIterNew(5, a, b, c, d, e) + return PyArray_MultiIterNew(5, a, b, + c, d, e) -cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: +cdef inline char* _util_dtypestring(dtype descr, char* f, + char* end, int* offset) except NULL: # Recursive utility function used in __getbuffer__ to get format # string. The new location in the format string is returned. @@ -800,7 +810,8 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset child, new_offset = fields if (end - f) - (new_offset - offset[0]) < 15: - raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") + raise RuntimeError(u"Format string allocated too short, " + "see comment in numpy.pxd") if ((child.byteorder == '>' and little_endian) or (child.byteorder == '<' and not little_endian)): @@ -860,7 +871,8 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset cdef extern from "numpy/ufuncobject.h": - ctypedef void (*PyUFuncGenericFunction) (char **, npy_intp *, npy_intp *, void *) + ctypedef void (*PyUFuncGenericFunction) (char **, npy_intp *, + npy_intp *, void *) ctypedef extern class numpy.ufunc [object PyUFuncObject]: cdef: @@ -968,14 +980,14 @@ cdef extern from "numpy/ufuncobject.h": cdef inline void set_array_base(ndarray arr, object base): - cdef PyObject* baseptr - if base is None: - baseptr = NULL - else: - Py_INCREF(base) # important to do this before decref below! - baseptr = base - Py_XDECREF(arr.base) - arr.base = baseptr + cdef PyObject* baseptr + if base is None: + baseptr = NULL + else: + Py_INCREF(base) # important to do this before decref below! + baseptr = base + Py_XDECREF(arr.base) + arr.base = baseptr cdef inline object get_array_base(ndarray arr): if arr.base is NULL: diff --git a/pandas/_libs/src/util.pxd b/pandas/_libs/src/util.pxd index f5fc684df24ce..84d6dddf338a5 100644 --- a/pandas/_libs/src/util.pxd +++ b/pandas/_libs/src/util.pxd @@ -115,7 +115,8 @@ cdef inline bint _checknull_old(object val): cdef double INF = np.inf cdef double NEGINF = -INF try: - return val is None or (cpython.PyFloat_Check(val) and (val != val or val == INF or val == NEGINF)) + return val is None or (cpython.PyFloat_Check(val) and + (val != val or val == INF or val == NEGINF)) except ValueError: return False From 5727dde2f0f2f56ef1f487958dcd250b9955e36b Mon Sep 17 00:00:00 2001 From: Manraj Singh Grover Date: Wed, 8 Nov 2017 23:57:13 +0530 Subject: [PATCH 4/6] Fix numpy.pxd #object error --- pandas/_libs/src/numpy.pxd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/src/numpy.pxd b/pandas/_libs/src/numpy.pxd index b7970ed6ae373..40a754164bbdd 100644 --- a/pandas/_libs/src/numpy.pxd +++ b/pandas/_libs/src/numpy.pxd @@ -605,8 +605,8 @@ cdef extern from "numpy/arrayobject.h": void PyArray_UpdateFlags (ndarray, int) object PyArray_New (type, int, npy_intp *, int, npy_intp *, void *, int, int, object) - #object PyArray_NewFromDescr ( - type, dtype, int, npy_intp *, npy_intp *, void *, int, object) + #object PyArray_NewFromDescr \ + (type, dtype, int, npy_intp *, npy_intp *, void *, int, object) #dtype PyArray_DescrNew (dtype) dtype PyArray_DescrNewFromType (int) double PyArray_GetPriority (object, double) From 0d73d951aaa42f072c7f5f884f8129b408e15113 Mon Sep 17 00:00:00 2001 From: Manraj Singh Grover Date: Thu, 9 Nov 2017 06:09:49 +0530 Subject: [PATCH 5/6] Remove commented lines in numpy.pxd --- pandas/_libs/src/numpy.pxd | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/_libs/src/numpy.pxd b/pandas/_libs/src/numpy.pxd index 40a754164bbdd..3d692614b04ce 100644 --- a/pandas/_libs/src/numpy.pxd +++ b/pandas/_libs/src/numpy.pxd @@ -605,8 +605,6 @@ cdef extern from "numpy/arrayobject.h": void PyArray_UpdateFlags (ndarray, int) object PyArray_New (type, int, npy_intp *, int, npy_intp *, void *, int, int, object) - #object PyArray_NewFromDescr \ - (type, dtype, int, npy_intp *, npy_intp *, void *, int, object) #dtype PyArray_DescrNew (dtype) dtype PyArray_DescrNewFromType (int) double PyArray_GetPriority (object, double) From 4b7caa6f6f0d8441ed5fb34e034cf5316f221a59 Mon Sep 17 00:00:00 2001 From: Manraj Singh Grover Date: Thu, 9 Nov 2017 17:12:03 +0530 Subject: [PATCH 6/6] 18121: Fix travis build --- pandas/_libs/src/numpy.pxd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/src/numpy.pxd b/pandas/_libs/src/numpy.pxd index 3d692614b04ce..6fa2bc6af9d1f 100644 --- a/pandas/_libs/src/numpy.pxd +++ b/pandas/_libs/src/numpy.pxd @@ -808,8 +808,8 @@ cdef inline char* _util_dtypestring(dtype descr, char* f, child, new_offset = fields if (end - f) - (new_offset - offset[0]) < 15: - raise RuntimeError(u"Format string allocated too short, " - "see comment in numpy.pxd") + raise RuntimeError( + u"Format string allocated too short, see comment in numpy.pxd") if ((child.byteorder == '>' and little_endian) or (child.byteorder == '<' and not little_endian)): @@ -980,10 +980,10 @@ cdef extern from "numpy/ufuncobject.h": cdef inline void set_array_base(ndarray arr, object base): cdef PyObject* baseptr if base is None: - baseptr = NULL + baseptr = NULL else: - Py_INCREF(base) # important to do this before decref below! - baseptr = base + Py_INCREF(base) # important to do this before decref below! + baseptr = base Py_XDECREF(arr.base) arr.base = baseptr