Merge remote-tracking branch 'upstream/master' into string-use-inf-as-na

Author: dsaxton

asv_bench/benchmarks/rolling.py

@@ -150,19 +150,18 @@ def time_quantile(self, constructor, window, dtype, percentile, interpolation):
         self.roll.quantile(percentile, interpolation=interpolation)
 
 
-class PeakMemFixed:
-    def setup(self):
-        N = 10
-        arr = 100 * np.random.random(N)
-        self.roll = pd.Series(arr).rolling(10)
-
-    def peakmem_fixed(self):
-        # GH 25926
-        # This is to detect memory leaks in rolling operations.
-        # To save time this is only ran on one method.
-        # 6000 iterations is enough for most types of leaks to be detected
-        for x in range(6000):
-            self.roll.max()
+class PeakMemFixedWindowMinMax:
+
+    params = ["min", "max"]
+
+    def setup(self, operation):
+        N = int(1e6)
+        arr = np.random.random(N)
+        self.roll = pd.Series(arr).rolling(2)
+
+    def peakmem_fixed(self, operation):
+        for x in range(5):
+            getattr(self.roll, operation)()
 
 
 class ForwardWindowMethods:

doc/source/whatsnew/v1.1.0.rst

@@ -698,6 +698,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.resample` where an ``AmbiguousTimeError`` would be raised when the resulting timezone aware :class:`DatetimeIndex` had a DST transition at midnight (:issue:`25758`)
 - Bug in :meth:`DataFrame.groupby` where a ``ValueError`` would be raised when grouping by a categorical column with read-only categories and ``sort=False`` (:issue:`33410`)
 - Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
+- Bug in :meth:`Rolling.min` and :meth:`Rolling.max`: Growing memory usage after multiple calls when using a fixed window (:issue:`30726`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/hashtable_class_helper.pxi.in

@@ -12,9 +12,6 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 from pandas._libs.tslibs.util cimport get_c_string
 from pandas._libs.missing cimport C_NA
 
-cdef extern from "Python.h":
-    void PyErr_Clear()
-
 {{py:
 
 # name, dtype, c_type
@@ -792,9 +789,9 @@ cdef class StringHashTable(HashTable):
                 labels[i] = na_sentinel
             else:
                 # if ignore_na is False, we also stringify NaN/None/etc.
-                v = get_c_string(<str>val)
-                if v == NULL:
-                    PyErr_Clear()
+                try:
+                    v = get_c_string(<str>val)
+                except UnicodeEncodeError:
                     v = get_c_string(<str>repr(val))
                 vecs[i] = v
 

pandas/_libs/tslibs/util.pxd

@@ -219,7 +219,7 @@ cdef inline bint is_nan(object val):
 
 
 cdef inline const char* get_c_string_buf_and_size(str py_string,
-                                                  Py_ssize_t *length):
+                                                  Py_ssize_t *length) except NULL:
     """
     Extract internal char* buffer of unicode or bytes object `py_string` with
     getting length of this internal buffer saved in `length`.
@@ -238,12 +238,8 @@ cdef inline const char* get_c_string_buf_and_size(str py_string,
     -------
     buf : const char*
     """
-    cdef:
-        const char *buf
+    return PyUnicode_AsUTF8AndSize(py_string, length)
 
-    buf = PyUnicode_AsUTF8AndSize(py_string, length)
-    return buf
 
-
-cdef inline const char* get_c_string(str py_string):
+cdef inline const char* get_c_string(str py_string) except NULL:
     return get_c_string_buf_and_size(py_string, NULL)

pandas/_libs/window/aggregations.pyx

@@ -971,8 +971,8 @@ cdef inline numeric calc_mm(int64_t minp, Py_ssize_t nobs,
     return result
 
 
-def roll_max_fixed(ndarray[float64_t] values, ndarray[int64_t] start,
-                   ndarray[int64_t] end, int64_t minp, int64_t win):
+def roll_max_fixed(float64_t[:] values, int64_t[:] start,
+                   int64_t[:] end, int64_t minp, int64_t win):
     """
     Moving max of 1d array of any numeric type along axis=0 ignoring NaNs.
 
@@ -988,7 +988,7 @@ def roll_max_fixed(ndarray[float64_t] values, ndarray[int64_t] start,
             make the interval closed on the right, left,
             both or neither endpoints
     """
-    return _roll_min_max_fixed(values, start, end, minp, win, is_max=1)
+    return _roll_min_max_fixed(values, minp, win, is_max=1)
 
 
 def roll_max_variable(ndarray[float64_t] values, ndarray[int64_t] start,
@@ -1011,8 +1011,8 @@ def roll_max_variable(ndarray[float64_t] values, ndarray[int64_t] start,
     return _roll_min_max_variable(values, start, end, minp, is_max=1)
 
 
-def roll_min_fixed(ndarray[float64_t] values, ndarray[int64_t] start,
-                   ndarray[int64_t] end, int64_t minp, int64_t win):
+def roll_min_fixed(float64_t[:] values, int64_t[:] start,
+                   int64_t[:] end, int64_t minp, int64_t win):
     """
     Moving min of 1d array of any numeric type along axis=0 ignoring NaNs.
 
@@ -1025,7 +1025,7 @@ def roll_min_fixed(ndarray[float64_t] values, ndarray[int64_t] start,
     index : ndarray, optional
        index for window computation
     """
-    return _roll_min_max_fixed(values, start, end, minp, win, is_max=0)
+    return _roll_min_max_fixed(values, minp, win, is_max=0)
 
 
 def roll_min_variable(ndarray[float64_t] values, ndarray[int64_t] start,
@@ -1112,9 +1112,7 @@ cdef _roll_min_max_variable(ndarray[numeric] values,
     return output
 
 
-cdef _roll_min_max_fixed(ndarray[numeric] values,
-                         ndarray[int64_t] starti,
-                         ndarray[int64_t] endi,
+cdef _roll_min_max_fixed(numeric[:] values,
                          int64_t minp,
                          int64_t win,
                          bint is_max):

pandas/tests/io/parser/test_common.py

@@ -974,6 +974,15 @@ def test_no_permission(all_parsers):
     msg = r"\[Errno 13\]"
     with tm.ensure_clean() as path:
         os.chmod(path, 0)  # make file unreadable
+
+        # verify that this process cannot open the file (not running as sudo)
+        try:
+            with open(path):
+                pass
+            pytest.skip("Running as sudo.")
+        except PermissionError:
+            pass
+
         with pytest.raises(PermissionError, match=msg) as e:
             parser.read_csv(path)
         assert path == e.value.filename