Skip to content

PERF: Speed up Spearman calculation #28151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Sep 7, 2019
19 changes: 15 additions & 4 deletions pandas/_libs/algos.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,7 @@ def nancorr_spearman(const float64_t[:, :] mat, Py_ssize_t minp=1):
cdef:
Py_ssize_t i, j, xi, yi, N, K
ndarray[float64_t, ndim=2] result
ndarray[float64_t, ndim=2] ranked_mat
ndarray[float64_t, ndim=1] maskedx
ndarray[float64_t, ndim=1] maskedy
ndarray[uint8_t, ndim=2] mask
Expand All @@ -307,10 +308,17 @@ def nancorr_spearman(const float64_t[:, :] mat, Py_ssize_t minp=1):
result = np.empty((K, K), dtype=np.float64)
mask = np.isfinite(mat).view(np.uint8)

ranked_mat = np.empty((N, K), dtype=np.float64)

for i in range(K):
ranked_mat[:, i] = rank_1d_float64(mat[:, i])

for xi in range(K):
for yi in range(xi + 1):
nobs = 0
same_miss_pat = True
for i in range(N):
same_miss_pat &= not (mask[i, xi] ^ mask[i, yi])
if mask[i, xi] and mask[i, yi]:
nobs += 1

Expand All @@ -320,13 +328,16 @@ def nancorr_spearman(const float64_t[:, :] mat, Py_ssize_t minp=1):
maskedx = np.empty(nobs, dtype=np.float64)
maskedy = np.empty(nobs, dtype=np.float64)
j = 0

for i in range(N):
if mask[i, xi] and mask[i, yi]:
maskedx[j] = mat[i, xi]
maskedy[j] = mat[i, yi]
maskedx[j] = ranked_mat[i, xi]
maskedy[j] = ranked_mat[i, yi]
j += 1
maskedx = rank_1d_float64(maskedx)
maskedy = rank_1d_float64(maskedy)

if not same_miss_pat:
maskedx = rank_1d_float64(maskedx)
maskedy = rank_1d_float64(maskedy)

mean = (nobs + 1) / 2.

Expand Down