From 27f4cc10b44c1b5acdcf5bcdc5e2eaeba102cd39 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 10:05:33 -0500 Subject: [PATCH 1/9] no build warnings --- pandas/_libs/hashtable_class_helper.pxi.in | 4 ++-- pandas/_libs/src/ujson/python/objToJSON.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/hashtable_class_helper.pxi.in b/pandas/_libs/hashtable_class_helper.pxi.in index 6671375f628e7..15fca0fc6ffc4 100644 --- a/pandas/_libs/hashtable_class_helper.pxi.in +++ b/pandas/_libs/hashtable_class_helper.pxi.in @@ -192,7 +192,7 @@ cdef class StringVector: append_data_string(self.data, x) - cdef extend(self, ndarray[:] x): + cdef extend(self, ndarray[object] x): for i in range(len(x)): self.append(x[i]) @@ -237,7 +237,7 @@ cdef class ObjectVector: self.external_view_exists = True return self.ao - cdef extend(self, ndarray[:] x): + cdef extend(self, ndarray[object] x): for i in range(len(x)): self.append(x[i]) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 8cfc20ffd2c1c..a982928061384 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -1454,7 +1454,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc, 1000000000LL; // nanoseconds per second } else { // datetime.* objects don't follow above rules - nanosecVal = PyDateTimeToEpoch(item, NPY_FR_ns); + nanosecVal = PyDateTimeToEpoch((PyDateTime_Date *)item, NPY_FR_ns); } } } From 965f7ecd71a6c77cbc0c94621ef442c47c74a83b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 10:09:49 -0500 Subject: [PATCH 2/9] Updated setup --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2d49d7e1e85f2..c5092b3d15da5 100755 --- a/setup.py +++ b/setup.py @@ -433,8 +433,7 @@ def run(self): extra_compile_args.append("/Z7") extra_link_args.append("/DEBUG") else: - # args to ignore warnings - extra_compile_args = [] + extra_compile_args = ["-Werror"] extra_link_args = [] if debugging_symbols_requested: extra_compile_args.append("-g") From 46128207616c4edeb6c9687046041f7e0568e394 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 12:54:43 -0800 Subject: [PATCH 3/9] uninitialized errors --- pandas/_libs/writers.pyx | 2 +- pandas/io/sas/sas.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/writers.pyx b/pandas/_libs/writers.pyx index 73201e75c3c88..985a6ea8318eb 100644 --- a/pandas/_libs/writers.pyx +++ b/pandas/_libs/writers.pyx @@ -31,7 +31,7 @@ def write_csv_rows(list data, ndarray data_index, """ # In crude testing, N>100 yields little marginal improvement cdef: - Py_ssize_t i, j, k = len(data_index), N = 100, ncols = len(cols) + Py_ssize_t i, j = 0, k = len(data_index), N = 100, ncols = len(cols) list rows # pre-allocate rows diff --git a/pandas/io/sas/sas.pyx b/pandas/io/sas/sas.pyx index 40fea0aaf0d07..11171af1e0c82 100644 --- a/pandas/io/sas/sas.pyx +++ b/pandas/io/sas/sas.pyx @@ -120,7 +120,7 @@ cdef const uint8_t[:] rdc_decompress(int result_length, const uint8_t[:] inbuff) cdef: uint8_t cmd - uint16_t ctrl_bits, ctrl_mask = 0, ofs, cnt + uint16_t ctrl_bits = 0, ctrl_mask = 0, ofs, cnt int rpos = 0, k uint8_t[:] outbuff = np.zeros(result_length, dtype=np.uint8) Py_ssize_t ipos = 0, length = len(inbuff) From 86494d341ae7c41e6badb154c2fb334c8370e814 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 13:11:17 -0800 Subject: [PATCH 4/9] more uninitialized --- pandas/_libs/internals.pyx | 7 ++----- pandas/_libs/parsers.pyx | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pandas/_libs/internals.pyx b/pandas/_libs/internals.pyx index 8bbbc6db94842..8a8da8c51796f 100644 --- a/pandas/_libs/internals.pyx +++ b/pandas/_libs/internals.pyx @@ -371,8 +371,8 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True): Py_ssize_t i, start, stop, n, diff object blkno - list group_order - dict group_dict + list group_order = [] + dict group_dict = {} int64_t[:] res_view n = blknos.shape[0] @@ -393,9 +393,6 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True): yield cur_blkno, slice(start, n) else: - group_order = [] - group_dict = {} - for i in range(1, n): if blknos[i] != cur_blkno: if cur_blkno not in group_dict: diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index 3077f73a8d1a4..15036a94b3ae6 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -701,7 +701,7 @@ cdef class TextReader: char *word object name, old_name int status - uint64_t hr, data_line + uint64_t hr, data_line = 0 char *errors = "strict" StringPath path = _string_path(self.c_encoding) @@ -805,7 +805,6 @@ cdef class TextReader: self._tokenize_rows(1) header = [ self.names ] - data_line = 0 if self.parser.lines < 1: field_count = len(header[0]) From 01cb64afe950f816d147cab0a7f754e5319d2204 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 13:23:42 -0800 Subject: [PATCH 5/9] np warnings compat --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c5092b3d15da5..5cb52d421b1fc 100755 --- a/setup.py +++ b/setup.py @@ -475,7 +475,10 @@ def run(self): # in numpy>=1.16.0, silence build warnings about deprecated API usage # we can't do anything about these warnings because they stem from # cython+numpy version mismatches. -macros.append(("NPY_NO_DEPRECATED_API", "0")) +if np.__version__ > LooseVersion("1.16.0"): + macros.append(("NPY_NO_DEPRECATED_API", "0")) +elif "-Werror" in extra_compile_args: + extra_compile_args.remove("-Werror") # ---------------------------------------------------------------------- From 6d3d543934bbd20275ad083f15a4f5d3bebf4127 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 13:33:10 -0800 Subject: [PATCH 6/9] numpy hackery --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 5cb52d421b1fc..d3ce7ca6aed95 100755 --- a/setup.py +++ b/setup.py @@ -475,10 +475,15 @@ def run(self): # in numpy>=1.16.0, silence build warnings about deprecated API usage # we can't do anything about these warnings because they stem from # cython+numpy version mismatches. -if np.__version__ > LooseVersion("1.16.0"): - macros.append(("NPY_NO_DEPRECATED_API", "0")) -elif "-Werror" in extra_compile_args: - extra_compile_args.remove("-Werror") +macros.append(("NPY_NO_DEPRECATED_API", "0")) +if "-Werror" in extra_compile_args: + try: + import numpy as np + except ImportError: + pass + else: + if np.__version__ > LooseVersion("1.16.0"): + extra_compile_args.remove("-Werror") # ---------------------------------------------------------------------- From 5c1fc8ab2e9bb0bda5c691e3bca64351ffff31f2 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 21 Feb 2020 13:38:55 -0800 Subject: [PATCH 7/9] typo fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3ce7ca6aed95..461ef005c3df3 100755 --- a/setup.py +++ b/setup.py @@ -482,7 +482,7 @@ def run(self): except ImportError: pass else: - if np.__version__ > LooseVersion("1.16.0"): + if np.__version__ < LooseVersion("1.16.0"): extra_compile_args.remove("-Werror") From 873230e6b713ac6ea3234ad36aff28e9042670fc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 18 Mar 2020 18:39:03 -0700 Subject: [PATCH 8/9] no warnings from internals --- pandas/_libs/internals.pyx | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/pandas/_libs/internals.pyx b/pandas/_libs/internals.pyx index 63f076b7ee993..c65205e406607 100644 --- a/pandas/_libs/internals.pyx +++ b/pandas/_libs/internals.pyx @@ -378,25 +378,23 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True): object blkno object group_dict = defaultdict(list) - int64_t[:] res_view n = blknos.shape[0] - - if n == 0: - return - + result = list() start = 0 cur_blkno = blknos[start] - if group is False: + if n == 0: + pass + elif group is False: for i in range(1, n): if blknos[i] != cur_blkno: - yield cur_blkno, slice(start, i) + result.append((cur_blkno, slice(start, i))) start = i cur_blkno = blknos[i] - yield cur_blkno, slice(start, n) + result.append((cur_blkno, slice(start, n))) else: for i in range(1, n): if blknos[i] != cur_blkno: @@ -409,19 +407,20 @@ def get_blkno_indexers(int64_t[:] blknos, bint group=True): for blkno, slices in group_dict.items(): if len(slices) == 1: - yield blkno, slice(slices[0][0], slices[0][1]) + result.append((blkno, slice(slices[0][0], slices[0][1]))) else: tot_len = sum(stop - start for start, stop in slices) - result = np.empty(tot_len, dtype=np.int64) - res_view = result + arr = np.empty(tot_len, dtype=np.int64) i = 0 for start, stop in slices: for diff in range(start, stop): - res_view[i] = diff + arr[i] = diff i += 1 - yield blkno, result + result.append((blkno, arr)) + + return result def get_blkno_placements(blknos, group: bool = True): From 5fc220c077605a9cc4942f94902c1f1201cf644c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 18 Mar 2020 19:33:14 -0700 Subject: [PATCH 9/9] No PyDateTime_Date usage --- pandas/_libs/src/ujson/python/date_conversions.c | 4 ++-- pandas/_libs/src/ujson/python/date_conversions.h | 5 ++--- pandas/_libs/src/ujson/python/objToJSON.c | 9 ++++----- pandas/_libs/tslibs/src/datetime/np_datetime.c | 10 ++++------ pandas/_libs/tslibs/src/datetime/np_datetime.h | 3 +-- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/pandas/_libs/src/ujson/python/date_conversions.c b/pandas/_libs/src/ujson/python/date_conversions.c index bcb1334d978ef..4c25ab572bebe 100644 --- a/pandas/_libs/src/ujson/python/date_conversions.c +++ b/pandas/_libs/src/ujson/python/date_conversions.c @@ -67,7 +67,7 @@ npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base) { } /* Convert PyDatetime To ISO C-string. mutates len */ -char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base, +char *PyDateTimeToIso(PyObject *obj, NPY_DATETIMEUNIT base, size_t *len) { npy_datetimestruct dts; int ret; @@ -98,7 +98,7 @@ char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base, return result; } -npy_datetime PyDateTimeToEpoch(PyDateTime_Date *dt, NPY_DATETIMEUNIT base) { +npy_datetime PyDateTimeToEpoch(PyObject *dt, NPY_DATETIMEUNIT base) { npy_datetimestruct dts; int ret; diff --git a/pandas/_libs/src/ujson/python/date_conversions.h b/pandas/_libs/src/ujson/python/date_conversions.h index 1b5cbf2a7e307..23e36999be43f 100644 --- a/pandas/_libs/src/ujson/python/date_conversions.h +++ b/pandas/_libs/src/ujson/python/date_conversions.h @@ -4,7 +4,6 @@ #define PY_SSIZE_T_CLEAN #include #include -#include "datetime.h" // Scales value inplace from nanosecond resolution to unit resolution int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit); @@ -23,10 +22,10 @@ npy_datetime NpyDateTimeToEpoch(npy_datetime dt, NPY_DATETIMEUNIT base); // up to precision `base` e.g. base="s" yields 2020-01-03T00:00:00Z // while base="ns" yields "2020-01-01T00:00:00.000000000Z" // len is mutated to save the length of the returned string -char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base, size_t *len); +char *PyDateTimeToIso(PyObject *obj, NPY_DATETIMEUNIT base, size_t *len); // Convert a Python Date/Datetime to Unix epoch with resolution base -npy_datetime PyDateTimeToEpoch(PyDateTime_Date *dt, NPY_DATETIMEUNIT base); +npy_datetime PyDateTimeToEpoch(PyObject *dt, NPY_DATETIMEUNIT base); char *int64ToIsoDuration(int64_t value, size_t *len); diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index 95e98779c2368..965d6aec2c1cf 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -1451,7 +1451,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc, } else { // datetime.* objects don't follow above rules nanosecVal = - PyDateTimeToEpoch((PyDateTime_Date *)item, NPY_FR_ns); + PyDateTimeToEpoch(item, NPY_FR_ns); } } } @@ -1469,8 +1469,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc, if (type_num == NPY_DATETIME) { cLabel = int64ToIso(nanosecVal, base, &len); } else { - cLabel = PyDateTimeToIso((PyDateTime_Date *)item, - base, &len); + cLabel = PyDateTimeToIso(item, base, &len); } } if (cLabel == NULL) { @@ -1683,7 +1682,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; GET_TC(tc)->longValue = - PyDateTimeToEpoch((PyDateTime_Date *)obj, base); + PyDateTimeToEpoch(obj, base); tc->type = JT_LONG; } return; @@ -1710,7 +1709,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) { NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit; GET_TC(tc)->longValue = - PyDateTimeToEpoch((PyDateTime_Date *)obj, base); + PyDateTimeToEpoch(obj, base); tc->type = JT_LONG; } return; diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.c b/pandas/_libs/tslibs/src/datetime/np_datetime.c index a8a47e2e90f93..f647098140528 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.c +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.c @@ -21,7 +21,6 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt #endif // NPY_NO_DEPRECATED_API #include -#include #include #include @@ -313,15 +312,14 @@ int cmp_npy_datetimestruct(const npy_datetimestruct *a, * object into a NumPy npy_datetimestruct. Uses tzinfo (if present) * to convert to UTC time. * - * While the C API has PyDate_* and PyDateTime_* functions, the following - * implementation just asks for attributes, and thus supports - * datetime duck typing. The tzinfo time zone conversion would require - * this style of access anyway. + * The following implementation just asks for attributes, and thus + * supports datetime duck typing. The tzinfo time zone conversion + * requires this style of access as well. * * Returns -1 on error, 0 on success, and 1 (with no error set) * if obj doesn't have the needed date or datetime attributes. */ -int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, +int convert_pydatetime_to_datetimestruct(PyObject *dtobj, npy_datetimestruct *out) { // Assumes that obj is a valid datetime object PyObject *tmp; diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.h b/pandas/_libs/tslibs/src/datetime/np_datetime.h index 549d38409ca83..0bbc24ed822c5 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.h +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.h @@ -22,7 +22,6 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt #endif // NPY_NO_DEPRECATED_API #include -#include typedef struct { npy_int64 days; @@ -35,7 +34,7 @@ extern const npy_datetimestruct _NS_MAX_DTS; // stuff pandas needs // ---------------------------------------------------------------------------- -int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, +int convert_pydatetime_to_datetimestruct(PyObject *dtobj, npy_datetimestruct *out); npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,