From e6dde8061318fa78b2784aee43a872a12194afb0 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Sun, 8 Jul 2018 11:00:28 -0500 Subject: [PATCH 01/12] Appveyor 3.7 --- appveyor.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index f70fc829ec971..1fd07401d3335 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,6 +25,12 @@ environment: CONDA_PY: "36" CONDA_NPY: "113" + - CONDA_ROOT: "C:\\Miniconda3_64" + PYTHON_VERSION: "3.7" + PYTHON_ARCH: "64" + CONDA_PY: "37" + CONDA_NPY: "113" + - CONDA_ROOT: "C:\\Miniconda3_64" PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" From e83d43b0c510abb2d9c17ec90c6d0031a16171fa Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Sun, 8 Jul 2018 11:07:43 -0500 Subject: [PATCH 02/12] ci package list --- appveyor.yml | 2 +- ci/appveyor-37.yaml | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 ci/appveyor-37.yaml diff --git a/appveyor.yml b/appveyor.yml index 1fd07401d3335..cfac91ca34a3a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ environment: PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" CONDA_PY: "37" - CONDA_NPY: "113" + CONDA_NPY: "114" - CONDA_ROOT: "C:\\Miniconda3_64" PYTHON_VERSION: "2.7" diff --git a/ci/appveyor-37.yaml b/ci/appveyor-37.yaml new file mode 100644 index 0000000000000..63e45d0544ad9 --- /dev/null +++ b/ci/appveyor-37.yaml @@ -0,0 +1,27 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - blosc + - bottleneck + - fastparquet + - feather-format + - matplotlib + - numexpr + - numpy=1.14* + - openpyxl + - pyarrow + - pytables + - python-dateutil + - python=3.6.* + - pytz + - scipy + - thrift=0.10* + - xlrd + - xlsxwriter + - xlwt + # universal + - cython>=0.28.2 + - pytest + - pytest-xdist From 9592b6d066705928c12346be3c70328a4c5e6711 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Sun, 8 Jul 2018 14:25:11 -0500 Subject: [PATCH 03/12] change image type --- appveyor.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cfac91ca34a3a..c6199c1493f22 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,18 +20,14 @@ environment: matrix: - CONDA_ROOT: "C:\\Miniconda3_64" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 PYTHON_VERSION: "3.6" PYTHON_ARCH: "64" CONDA_PY: "36" CONDA_NPY: "113" - CONDA_ROOT: "C:\\Miniconda3_64" - PYTHON_VERSION: "3.7" - PYTHON_ARCH: "64" - CONDA_PY: "37" - CONDA_NPY: "114" - - - CONDA_ROOT: "C:\\Miniconda3_64" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" CONDA_PY: "27" From 2305a3215ceb49c08d4ce065e20d4741b6c881bb Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Mon, 9 Jul 2018 19:29:23 -0500 Subject: [PATCH 04/12] try hack fix --- pandas/_libs/window.pyx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index c52bd99c37e31..ad108435af478 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -653,16 +653,21 @@ cdef inline void add_var(double val, double *nobs, double *mean_x, double *ssqdm_x) nogil: """ add a value from the var calc """ cdef double delta - + nobs[0] = nobs[0] + 1 # Not NaN if val == val: - nobs[0] = nobs[0] + 1 - # a part of Welford's method for the online variance-calculation # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance delta = val - mean_x[0] mean_x[0] = mean_x[0] + delta / nobs[0] ssqdm_x[0] = ssqdm_x[0] + ((nobs[0] - 1) * delta ** 2) / nobs[0] + else: + # XXX + # `nobs[0] = nobs[0] + 1` should be in the if branch + # but something goes wrong with MSVC 2017 causing the whole + # path to optimize out, uncoditionally adding and + # backing out as a hack to fix + nobs[0] = nobs[0] - 1 cdef inline void remove_var(double val, double *nobs, double *mean_x, @@ -683,7 +688,6 @@ cdef inline void remove_var(double val, double *nobs, double *mean_x, mean_x[0] = 0 ssqdm_x[0] = 0 - def roll_var(ndarray[double_t] input, int64_t win, int64_t minp, object index, object closed, int ddof=1): """ From cfc5c29b31d5dad2bb78f646e8f83ff0cfa29eea Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Tue, 10 Jul 2018 19:02:01 -0500 Subject: [PATCH 05/12] lint --- ci/appveyor-37.yaml | 27 --------------------------- pandas/_libs/window.pyx | 1 + 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 ci/appveyor-37.yaml diff --git a/ci/appveyor-37.yaml b/ci/appveyor-37.yaml deleted file mode 100644 index 63e45d0544ad9..0000000000000 --- a/ci/appveyor-37.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: pandas -channels: - - defaults - - conda-forge -dependencies: - - blosc - - bottleneck - - fastparquet - - feather-format - - matplotlib - - numexpr - - numpy=1.14* - - openpyxl - - pyarrow - - pytables - - python-dateutil - - python=3.6.* - - pytz - - scipy - - thrift=0.10* - - xlrd - - xlsxwriter - - xlwt - # universal - - cython>=0.28.2 - - pytest - - pytest-xdist diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index ad108435af478..cad932922edf9 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -688,6 +688,7 @@ cdef inline void remove_var(double val, double *nobs, double *mean_x, mean_x[0] = 0 ssqdm_x[0] = 0 + def roll_var(ndarray[double_t] input, int64_t win, int64_t minp, object index, object closed, int ddof=1): """ From 02779e399b18a335211e690626ed93af63e1e0a5 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Sun, 15 Jul 2018 13:48:58 -0500 Subject: [PATCH 06/12] use isnan on problem function --- pandas/_libs/window.pyx | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index cad932922edf9..da433299de510 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -6,6 +6,7 @@ from cython cimport Py_ssize_t from libcpp.deque cimport deque from libc.stdlib cimport malloc, free +from libc.math cimport isnan import numpy as np cimport numpy as cnp @@ -649,25 +650,20 @@ cdef inline double calc_var(int64_t minp, int ddof, double nobs, return result -cdef inline void add_var(double val, double *nobs, double *mean_x, +cdef inline void add_var(const double val, double *nobs, double *mean_x, double *ssqdm_x) nogil: """ add a value from the var calc """ cdef double delta + # `isnan` instead of equality as fix for GH-21813 + if isnan(val): + return + nobs[0] = nobs[0] + 1 - # Not NaN - if val == val: - # a part of Welford's method for the online variance-calculation - # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance - delta = val - mean_x[0] - mean_x[0] = mean_x[0] + delta / nobs[0] - ssqdm_x[0] = ssqdm_x[0] + ((nobs[0] - 1) * delta ** 2) / nobs[0] - else: - # XXX - # `nobs[0] = nobs[0] + 1` should be in the if branch - # but something goes wrong with MSVC 2017 causing the whole - # path to optimize out, uncoditionally adding and - # backing out as a hack to fix - nobs[0] = nobs[0] - 1 + # a part of Welford's method for the online variance-calculation + # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + delta = val - mean_x[0] + mean_x[0] = mean_x[0] + delta / nobs[0] + ssqdm_x[0] = ssqdm_x[0] + ((nobs[0] - 1) * delta ** 2) / nobs[0] cdef inline void remove_var(double val, double *nobs, double *mean_x, From 78de263ce7dbc9b2697676923e4205ca8aa1e191 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Mon, 16 Jul 2018 18:36:02 -0500 Subject: [PATCH 07/12] use numpy compat isnan --- pandas/_libs/window.pyx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index da433299de510..0f81b993f8ac5 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -6,7 +6,6 @@ from cython cimport Py_ssize_t from libcpp.deque cimport deque from libc.stdlib cimport malloc, free -from libc.math cimport isnan import numpy as np cimport numpy as cnp @@ -18,6 +17,9 @@ cdef extern from "src/headers/cmath" namespace "std": int signbit(double) nogil double sqrt(double x) nogil +cdef extern from "numpy/npy_math.h": + bint isnan(double) nogil + cimport util from util cimport numeric From ff2f3891ecbae810f97b39587f5296fcddf3cf8b Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Tue, 17 Jul 2018 07:07:18 -0500 Subject: [PATCH 08/12] use right isnan --- pandas/_libs/window.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 0f81b993f8ac5..b70c566b35672 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -18,7 +18,7 @@ cdef extern from "src/headers/cmath" namespace "std": double sqrt(double x) nogil cdef extern from "numpy/npy_math.h": - bint isnan(double) nogil + bint npy_isnan(double) nogil cimport util from util cimport numeric @@ -657,7 +657,7 @@ cdef inline void add_var(const double val, double *nobs, double *mean_x, """ add a value from the var calc """ cdef double delta # `isnan` instead of equality as fix for GH-21813 - if isnan(val): + if npy_isnan(val): return nobs[0] = nobs[0] + 1 From 87e69e824661a55ddcb623ec529cde7f09dfcbc7 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Wed, 18 Jul 2018 21:16:33 -0500 Subject: [PATCH 09/12] work around OSX math undefs --- doc/source/whatsnew/v0.23.4.txt | 2 +- pandas/_libs/src/headers/cmath | 1 + pandas/_libs/window.pyx | 8 +++----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.23.4.txt b/doc/source/whatsnew/v0.23.4.txt index 69525aaea1d62..c17f4ffdd6b8e 100644 --- a/doc/source/whatsnew/v0.23.4.txt +++ b/doc/source/whatsnew/v0.23.4.txt @@ -20,7 +20,7 @@ and bug fixes. We recommend that all users upgrade to this version. Fixed Regressions ~~~~~~~~~~~~~~~~~ -- +- Python 3.7 with Windows gave all missing values for rolling variance calculations (:issue:`21813`) - .. _whatsnew_0234.bug_fixes: diff --git a/pandas/_libs/src/headers/cmath b/pandas/_libs/src/headers/cmath index d8e2239406cae..b74109c5bb5aa 100644 --- a/pandas/_libs/src/headers/cmath +++ b/pandas/_libs/src/headers/cmath @@ -6,6 +6,7 @@ #if defined(_MSC_VER) && (_MSC_VER < 1800) #include namespace std { + #define isnan(x) _isnan(x) __inline int signbit(double num) { return _copysign(1.0, num) < 0; } } #else diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index b70c566b35672..dddcad07938b6 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -14,12 +14,10 @@ cnp.import_array() cdef extern from "src/headers/cmath" namespace "std": + bint isnan(double) nogil int signbit(double) nogil double sqrt(double x) nogil -cdef extern from "numpy/npy_math.h": - bint npy_isnan(double) nogil - cimport util from util cimport numeric @@ -656,8 +654,8 @@ cdef inline void add_var(const double val, double *nobs, double *mean_x, double *ssqdm_x) nogil: """ add a value from the var calc """ cdef double delta - # `isnan` instead of equality as fix for GH-21813 - if npy_isnan(val): + # `isnan` instead of equality as fix for GH-21813, msvc 2017 bug + if isnan(val): return nobs[0] = nobs[0] + 1 From f0628c0624af0e12c9f35788b35e86a82047a4d3 Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Wed, 18 Jul 2018 21:31:18 -0500 Subject: [PATCH 10/12] cleanup const --- pandas/_libs/window.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index dddcad07938b6..6453b5ed2ab3a 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -650,7 +650,7 @@ cdef inline double calc_var(int64_t minp, int ddof, double nobs, return result -cdef inline void add_var(const double val, double *nobs, double *mean_x, +cdef inline void add_var(double val, double *nobs, double *mean_x, double *ssqdm_x) nogil: """ add a value from the var calc """ cdef double delta From ec6fbe2d59477dd36bc95f8ca2692f18c8f5fbbe Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Sun, 22 Jul 2018 18:50:40 -0500 Subject: [PATCH 11/12] fix reversion --- pandas/_libs/src/headers/cmath | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/_libs/src/headers/cmath b/pandas/_libs/src/headers/cmath index b74109c5bb5aa..64292d284a28f 100644 --- a/pandas/_libs/src/headers/cmath +++ b/pandas/_libs/src/headers/cmath @@ -6,6 +6,7 @@ #if defined(_MSC_VER) && (_MSC_VER < 1800) #include namespace std { + __inline int isnan(double x) { return _isnan(x); } #define isnan(x) _isnan(x) __inline int signbit(double num) { return _copysign(1.0, num) < 0; } } From 5d40eaefe8c162d0170caf15dac5e9b15bffff6b Mon Sep 17 00:00:00 2001 From: Chris Bartak Date: Mon, 23 Jul 2018 06:59:03 -0500 Subject: [PATCH 12/12] ... --- pandas/_libs/src/headers/cmath | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/_libs/src/headers/cmath b/pandas/_libs/src/headers/cmath index 64292d284a28f..2bccf9bb13d77 100644 --- a/pandas/_libs/src/headers/cmath +++ b/pandas/_libs/src/headers/cmath @@ -7,7 +7,6 @@ #include namespace std { __inline int isnan(double x) { return _isnan(x); } - #define isnan(x) _isnan(x) __inline int signbit(double num) { return _copysign(1.0, num) < 0; } } #else