From d0dbe9308798603920b7750270f1f79d0f30f8b1 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 21 Nov 2017 17:37:08 -0800 Subject: [PATCH 1/4] simplify skiplist inclusion/cimport to be more cythonize-friendly --- pandas/_libs/src/skiplist.pxd | 22 ---------------------- pandas/_libs/src/skiplist.pyx | 26 ++++++++++++++++++++++++-- pandas/_libs/window.pyx | 15 ++++----------- setup.py | 2 +- 4 files changed, 29 insertions(+), 36 deletions(-) delete mode 100644 pandas/_libs/src/skiplist.pxd diff --git a/pandas/_libs/src/skiplist.pxd b/pandas/_libs/src/skiplist.pxd deleted file mode 100644 index 214aa1c7aeaf0..0000000000000 --- a/pandas/_libs/src/skiplist.pxd +++ /dev/null @@ -1,22 +0,0 @@ -cdef extern from "skiplist.h": - ctypedef struct node_t: - node_t **next - int *width - double value - int is_nil - int levels - int ref_count - - ctypedef struct skiplist_t: - node_t *head - node_t **tmp_chain - int *tmp_steps - int size - int maxlevels - - skiplist_t* skiplist_init(int) nogil - void skiplist_destroy(skiplist_t*) nogil - double skiplist_get(skiplist_t*, int, int*) nogil - int skiplist_insert(skiplist_t*, double) nogil - int skiplist_remove(skiplist_t*, double) nogil - diff --git a/pandas/_libs/src/skiplist.pyx b/pandas/_libs/src/skiplist.pyx index 1524dca38d0e0..704f4e65fcfeb 100644 --- a/pandas/_libs/src/skiplist.pyx +++ b/pandas/_libs/src/skiplist.pyx @@ -6,8 +6,30 @@ # Cython version: Wes McKinney -cdef extern from "math.h": - double log(double x) +cdef extern from "skiplist.h": + ctypedef struct node_t: + node_t **next + int *width + double value + int is_nil + int levels + int ref_count + + ctypedef struct skiplist_t: + node_t *head + node_t **tmp_chain + int *tmp_steps + int size + int maxlevels + + skiplist_t* skiplist_init(int) nogil + void skiplist_destroy(skiplist_t*) nogil + double skiplist_get(skiplist_t*, int, int*) nogil + int skiplist_insert(skiplist_t*, double) nogil + int skiplist_remove(skiplist_t*, double) nogil + + +from libc.math cimport log # MSVC does not have log2! diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 4d5ebdc0c581a..7411be5d07605 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -13,7 +13,11 @@ np.import_array() cimport util from libc.stdlib cimport malloc, free +from libc.math cimport sqrt +# from numpy.npy_math cimport npy_signbit as signbit +cdef extern from "numpy/npy_math.h": + int signbit "npy_signbit" (double) nogil from numpy cimport ndarray, double_t, int64_t, float64_t @@ -30,17 +34,6 @@ cdef inline int int_min(int a, int b): return a if a <= b else b from util cimport numeric -from skiplist cimport ( - skiplist_t, - skiplist_init, - skiplist_destroy, - skiplist_get, - skiplist_insert, - skiplist_remove) - -cdef extern from "../src/headers/math.h": - double sqrt(double x) nogil - int signbit(double) nogil include "skiplist.pyx" diff --git a/setup.py b/setup.py index 44e7de1665bf0..099d246e700ab 100755 --- a/setup.py +++ b/setup.py @@ -611,7 +611,7 @@ def pxd(name): 'pyxfile': '_libs/testing'}, '_libs.window': { 'pyxfile': '_libs/window', - 'pxdfiles': ['_libs/src/skiplist', '_libs/src/util'], + 'pxdfiles': ['_libs/src/util'], 'depends': ['pandas/_libs/src/skiplist.pyx', 'pandas/_libs/src/skiplist.h']}, 'io.sas._sas': { From 1e8012690734e43f517b50f62edfce927fa1ed48 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 21 Nov 2017 19:50:29 -0800 Subject: [PATCH 2/4] restore signbit as it broke on appveyor --- pandas/_libs/window.pyx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 7411be5d07605..c1971bf7ebd48 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -15,10 +15,6 @@ cimport util from libc.stdlib cimport malloc, free from libc.math cimport sqrt -# from numpy.npy_math cimport npy_signbit as signbit -cdef extern from "numpy/npy_math.h": - int signbit "npy_signbit" (double) nogil - from numpy cimport ndarray, double_t, int64_t, float64_t cdef np.float32_t MINfloat32 = np.NINF @@ -34,6 +30,8 @@ cdef inline int int_min(int a, int b): return a if a <= b else b from util cimport numeric +cdef extern from "../src/headers/math.h": + int signbit(double) nogil include "skiplist.pyx" From 0fde0459eaa84c35bc4d6fda6d640d9358ee626a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 24 Nov 2017 15:19:01 -0800 Subject: [PATCH 3/4] make skiplist a standalone extension --- pandas/_libs/skiplist.pxd | 48 +++++++++++++++++++++++++++++ pandas/_libs/{src => }/skiplist.pyx | 38 +++++------------------ pandas/_libs/window.pyx | 8 +++-- setup.py | 8 +++-- 4 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 pandas/_libs/skiplist.pxd rename pandas/_libs/{src => }/skiplist.pyx (85%) diff --git a/pandas/_libs/skiplist.pxd b/pandas/_libs/skiplist.pxd new file mode 100644 index 0000000000000..82a0862112199 --- /dev/null +++ b/pandas/_libs/skiplist.pxd @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# cython: profile=False + +from cython cimport Py_ssize_t + +from numpy cimport double_t + + +cdef extern from "src/skiplist.h": + ctypedef struct node_t: + node_t **next + int *width + double value + int is_nil + int levels + int ref_count + + ctypedef struct skiplist_t: + node_t *head + node_t **tmp_chain + int *tmp_steps + int size + int maxlevels + + skiplist_t* skiplist_init(int) nogil + void skiplist_destroy(skiplist_t*) nogil + double skiplist_get(skiplist_t*, int, int*) nogil + int skiplist_insert(skiplist_t*, double) nogil + int skiplist_remove(skiplist_t*, double) nogil + + +# Note: Node is declared here so that IndexableSkiplist can be exposed; +# Node itself not intended to be exposed. +cdef class Node: + cdef public: + double_t value + list next + list width + + +cdef class IndexableSkiplist: + cdef: + Py_ssize_t size, maxlevels + Node head + + cpdef get(self, Py_ssize_t i) + cpdef insert(self, double value) + cpdef remove(self, double value) diff --git a/pandas/_libs/src/skiplist.pyx b/pandas/_libs/skiplist.pyx similarity index 85% rename from pandas/_libs/src/skiplist.pyx rename to pandas/_libs/skiplist.pyx index 704f4e65fcfeb..c96413edfb0f2 100644 --- a/pandas/_libs/src/skiplist.pyx +++ b/pandas/_libs/skiplist.pyx @@ -6,29 +6,6 @@ # Cython version: Wes McKinney -cdef extern from "skiplist.h": - ctypedef struct node_t: - node_t **next - int *width - double value - int is_nil - int levels - int ref_count - - ctypedef struct skiplist_t: - node_t *head - node_t **tmp_chain - int *tmp_steps - int size - int maxlevels - - skiplist_t* skiplist_init(int) nogil - void skiplist_destroy(skiplist_t*) nogil - double skiplist_get(skiplist_t*, int, int*) nogil - int skiplist_insert(skiplist_t*, double) nogil - int skiplist_remove(skiplist_t*, double) nogil - - from libc.math cimport log # MSVC does not have log2! @@ -38,6 +15,7 @@ cdef double Log2(double x): cimport numpy as np import numpy as np +from numpy cimport double_t from random import random @@ -47,10 +25,10 @@ np.import_array() # TODO: optimize this, make less messy cdef class Node: - cdef public: - double_t value - list next - list width + # cdef public: + # double_t value + # list next + # list width def __init__(self, double_t value, list next, list width): self.value = value @@ -65,9 +43,9 @@ cdef class IndexableSkiplist: Sorted collection supporting O(lg n) insertion, removal, and lookup by rank. """ - cdef: - Py_ssize_t size, maxlevels - Node head + # cdef: + # Py_ssize_t size, maxlevels + # Node head def __init__(self, expected_size=100): self.size = 0 diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index c1971bf7ebd48..1ca31e638f470 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -17,6 +17,11 @@ from libc.math cimport sqrt from numpy cimport ndarray, double_t, int64_t, float64_t +from skiplist cimport (IndexableSkiplist, + node_t, skiplist_t, + skiplist_init, skiplist_destroy, + skiplist_get, skiplist_insert, skiplist_remove) + cdef np.float32_t MINfloat32 = np.NINF cdef np.float64_t MINfloat64 = np.NINF @@ -30,10 +35,9 @@ cdef inline int int_min(int a, int b): return a if a <= b else b from util cimport numeric -cdef extern from "../src/headers/math.h": +cdef extern from "src/headers/math.h": int signbit(double) nogil -include "skiplist.pyx" # Cython implementations of rolling sum, mean, variance, skewness, # other statistical moment functions diff --git a/setup.py b/setup.py index 1a67549ea7c01..37be0b696503d 100755 --- a/setup.py +++ b/setup.py @@ -341,6 +341,7 @@ class CheckSDist(sdist_class): 'pandas/_libs/missing.pyx', 'pandas/_libs/testing.pyx', 'pandas/_libs/window.pyx', + 'pandas/_libs/skiplist.pyx', 'pandas/_libs/sparse.pyx', 'pandas/_libs/parsers.pyx', 'pandas/_libs/tslibs/strptime.pyx', @@ -544,6 +545,9 @@ def pxd(name): '_libs.reshape': { 'pyxfile': '_libs/reshape', 'depends': _pxi_dep['reshape']}, + '_libs.skiplist': { + 'pyxfile': '_libs/skiplist', + 'depends': ['pandas/_libs/src/skiplist.h']}, '_libs.sparse': { 'pyxfile': '_libs/sparse', 'depends': _pxi_dep['sparse']}, @@ -629,9 +633,7 @@ def pxd(name): 'pyxfile': '_libs/testing'}, '_libs.window': { 'pyxfile': '_libs/window', - 'pxdfiles': ['_libs/src/util'], - 'depends': ['pandas/_libs/src/skiplist.pyx', - 'pandas/_libs/src/skiplist.h']}, + 'pxdfiles': ['_libs/skiplist', '_libs/src/util']}, 'io.sas._sas': { 'pyxfile': 'io/sas/sas'}} From 64d54822dfb5440279f36b37e66808cf741feba4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 26 Nov 2017 07:56:22 -0800 Subject: [PATCH 4/4] revert sqrt to use nogil version --- pandas/_libs/window.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 1ca31e638f470..f9552e204c461 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -13,7 +13,6 @@ np.import_array() cimport util from libc.stdlib cimport malloc, free -from libc.math cimport sqrt from numpy cimport ndarray, double_t, int64_t, float64_t @@ -35,8 +34,9 @@ cdef inline int int_min(int a, int b): return a if a <= b else b from util cimport numeric -cdef extern from "src/headers/math.h": +cdef extern from "../src/headers/math.h": int signbit(double) nogil + double sqrt(double x) nogil # Cython implementations of rolling sum, mean, variance, skewness,