Merge pull request #155 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

ci/run_tests.sh

@@ -29,3 +29,8 @@ fi
 
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
+
+PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS  --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
+
+echo $PYTEST_AM_CMD
+sh -c "$PYTEST_AM_CMD"

doc/source/development/contributing.rst

@@ -325,7 +325,11 @@ Creating a Python environment (pip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you aren't using conda for your development environment, follow these instructions.
-You'll need to have at least Python 3.6.1 installed on your system.
+You'll need to have at least Python 3.7.0 installed on your system. If your Python version
+is 3.8.0 (or later), you might need to update your ``setuptools`` to version 42.0.0 (or later)
+in your development environment before installing the build dependencies::
+
+      pip install --upgrade setuptools
 
 **Unix**/**macOS with virtualenv**
 

doc/source/whatsnew/v1.2.4.rst

@@ -18,6 +18,7 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.sum` when ``min_count`` greater than the :class:`DataFrame` shape was passed resulted in a ``ValueError`` (:issue:`39738`)
 - Fixed regression in :meth:`DataFrame.to_json` raising ``AttributeError`` when run on PyPy (:issue:`39837`)
 - Fixed regression in :meth:`DataFrame.where` not returning a copy in the case of an all True condition (:issue:`39595`)
+- Fixed regression in :meth:`DataFrame.replace` raising ``IndexError`` when ``regex`` was a multi-key dictionary (:issue:`39338`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/_libs/groupby.pyx

@@ -1249,26 +1249,30 @@ def group_min(groupby_t[:, ::1] out,
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def group_cummin(groupby_t[:, ::1] out,
-                 ndarray[groupby_t, ndim=2] values,
-                 const int64_t[:] labels,
-                 int ngroups,
-                 bint is_datetimelike):
+def group_cummin_max(groupby_t[:, ::1] out,
+                     ndarray[groupby_t, ndim=2] values,
+                     const int64_t[:] labels,
+                     int ngroups,
+                     bint is_datetimelike,
+                     bint compute_max):
     """
-    Cumulative minimum of columns of `values`, in row groups `labels`.
+    Cumulative minimum/maximum of columns of `values`, in row groups `labels`.
 
     Parameters
     ----------
     out : array
-        Array to store cummin in.
+        Array to store cummin/max in.
     values : array
-        Values to take cummin of.
+        Values to take cummin/max of.
     labels : int64 array
         Labels to group by.
     ngroups : int
         Number of groups, larger than all entries of `labels`.
     is_datetimelike : bool
         True if `values` contains datetime-like entries.
+    compute_max : bool
+        True if cumulative maximum should be computed, False
+        if cumulative minimum should be computed
 
     Notes
     -----
@@ -1283,11 +1287,11 @@ def group_cummin(groupby_t[:, ::1] out,
     N, K = (<object>values).shape
     accum = np.empty((ngroups, K), dtype=np.asarray(values).dtype)
     if groupby_t is int64_t:
-        accum[:] = _int64_max
+        accum[:] = -_int64_max if compute_max else _int64_max
     elif groupby_t is uint64_t:
-        accum[:] = np.iinfo(np.uint64).max
+        accum[:] = 0 if compute_max else np.iinfo(np.uint64).max
     else:
-        accum[:] = np.inf
+        accum[:] = -np.inf if compute_max else np.inf
 
     with nogil:
         for i in range(N):
@@ -1302,66 +1306,32 @@ def group_cummin(groupby_t[:, ::1] out,
                     out[i, j] = val
                 else:
                     mval = accum[lab, j]
-                    if val < mval:
-                        accum[lab, j] = mval = val
+                    if compute_max:
+                        if val > mval:
+                            accum[lab, j] = mval = val
+                    else:
+                        if val < mval:
+                            accum[lab, j] = mval = val
                     out[i, j] = mval
 
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def group_cummax(groupby_t[:, ::1] out,
+def group_cummin(groupby_t[:, ::1] out,
                  ndarray[groupby_t, ndim=2] values,
                  const int64_t[:] labels,
                  int ngroups,
                  bint is_datetimelike):
-    """
-    Cumulative maximum of columns of `values`, in row groups `labels`.
+    """See group_cummin_max.__doc__"""
+    group_cummin_max(out, values, labels, ngroups, is_datetimelike, compute_max=False)
 
-    Parameters
-    ----------
-    out : array
-        Array to store cummax in.
-    values : array
-        Values to take cummax of.
-    labels : int64 array
-        Labels to group by.
-    ngroups : int
-        Number of groups, larger than all entries of `labels`.
-    is_datetimelike : bool
-        True if `values` contains datetime-like entries.
 
-    Notes
-    -----
-    This method modifies the `out` parameter, rather than returning an object.
-    """
-    cdef:
-        Py_ssize_t i, j, N, K, size
-        groupby_t val, mval
-        ndarray[groupby_t, ndim=2] accum
-        int64_t lab
-
-    N, K = (<object>values).shape
-    accum = np.empty((ngroups, K), dtype=np.asarray(values).dtype)
-    if groupby_t is int64_t:
-        accum[:] = -_int64_max
-    elif groupby_t is uint64_t:
-        accum[:] = 0
-    else:
-        accum[:] = -np.inf
-
-    with nogil:
-        for i in range(N):
-            lab = labels[i]
-
-            if lab < 0:
-                continue
-            for j in range(K):
-                val = values[i, j]
-
-                if _treat_as_na(val, is_datetimelike):
-                    out[i, j] = val
-                else:
-                    mval = accum[lab, j]
-                    if val > mval:
-                        accum[lab, j] = mval = val
-                    out[i, j] = mval
+@cython.boundscheck(False)
+@cython.wraparound(False)
+def group_cummax(groupby_t[:, ::1] out,
+                 ndarray[groupby_t, ndim=2] values,
+                 const int64_t[:] labels,
+                 int ngroups,
+                 bint is_datetimelike):
+    """See group_cummin_max.__doc__"""
+    group_cummin_max(out, values, labels, ngroups, is_datetimelike, compute_max=True)

pandas/_libs/join.pyx

@@ -231,7 +231,7 @@ cdef ndarray[intp_t] _get_result_indexer(
     return res
 
 
-def ffill_indexer(const intp_t[:] indexer):
+def ffill_indexer(const intp_t[:] indexer) -> np.ndarray:
     cdef:
         Py_ssize_t i, n = len(indexer)
         ndarray[intp_t] result
@@ -275,15 +275,15 @@ ctypedef fused join_t:
 def left_join_indexer_unique(ndarray[join_t] left, ndarray[join_t] right):
     cdef:
         Py_ssize_t i, j, nleft, nright
-        ndarray[int64_t] indexer
+        ndarray[intp_t] indexer
         join_t lval, rval
 
     i = 0
     j = 0
     nleft = len(left)
     nright = len(right)
 
-    indexer = np.empty(nleft, dtype=np.int64)
+    indexer = np.empty(nleft, dtype=np.intp)
     while True:
         if i == nleft:
             break
@@ -324,7 +324,7 @@ def left_join_indexer(ndarray[join_t] left, ndarray[join_t] right):
     cdef:
         Py_ssize_t i, j, k, nright, nleft, count
         join_t lval, rval
-        ndarray[int64_t] lindexer, rindexer
+        ndarray[intp_t] lindexer, rindexer
         ndarray[join_t] result
 
     nleft = len(left)
@@ -366,8 +366,8 @@ def left_join_indexer(ndarray[join_t] left, ndarray[join_t] right):
 
     # do it again now that result size is known
 
-    lindexer = np.empty(count, dtype=np.int64)
-    rindexer = np.empty(count, dtype=np.int64)
+    lindexer = np.empty(count, dtype=np.intp)
+    rindexer = np.empty(count, dtype=np.intp)
     result = np.empty(count, dtype=left.dtype)
 
     i = 0
@@ -427,7 +427,7 @@ def inner_join_indexer(ndarray[join_t] left, ndarray[join_t] right):
     cdef:
         Py_ssize_t i, j, k, nright, nleft, count
         join_t lval, rval
-        ndarray[int64_t] lindexer, rindexer
+        ndarray[intp_t] lindexer, rindexer
         ndarray[join_t] result
 
     nleft = len(left)
@@ -468,8 +468,8 @@ def inner_join_indexer(ndarray[join_t] left, ndarray[join_t] right):
 
     # do it again now that result size is known
 
-    lindexer = np.empty(count, dtype=np.int64)
-    rindexer = np.empty(count, dtype=np.int64)
+    lindexer = np.empty(count, dtype=np.intp)
+    rindexer = np.empty(count, dtype=np.intp)
     result = np.empty(count, dtype=left.dtype)
 
     i = 0
@@ -517,7 +517,7 @@ def outer_join_indexer(ndarray[join_t] left, ndarray[join_t] right):
     cdef:
         Py_ssize_t i, j, nright, nleft, count
         join_t lval, rval
-        ndarray[int64_t] lindexer, rindexer
+        ndarray[intp_t] lindexer, rindexer
         ndarray[join_t] result
 
     nleft = len(left)
@@ -564,8 +564,8 @@ def outer_join_indexer(ndarray[join_t] left, ndarray[join_t] right):
                 count += 1
                 j += 1
 
-    lindexer = np.empty(count, dtype=np.int64)
-    rindexer = np.empty(count, dtype=np.int64)
+    lindexer = np.empty(count, dtype=np.intp)
+    rindexer = np.empty(count, dtype=np.intp)
     result = np.empty(count, dtype=left.dtype)
 
     # do it again, but populate the indexers / result
@@ -673,12 +673,12 @@ def asof_join_backward_on_X_by_Y(asof_t[:] left_values,
                                  asof_t[:] right_values,
                                  by_t[:] left_by_values,
                                  by_t[:] right_by_values,
-                                 bint allow_exact_matches=1,
+                                 bint allow_exact_matches=True,
                                  tolerance=None):
 
     cdef:
         Py_ssize_t left_pos, right_pos, left_size, right_size, found_right_pos
-        ndarray[int64_t] left_indexer, right_indexer
+        ndarray[intp_t] left_indexer, right_indexer
         bint has_tolerance = False
         asof_t tolerance_ = 0
         asof_t diff = 0
@@ -693,8 +693,8 @@ def asof_join_backward_on_X_by_Y(asof_t[:] left_values,
     left_size = len(left_values)
     right_size = len(right_values)
 
-    left_indexer = np.empty(left_size, dtype=np.int64)
-    right_indexer = np.empty(left_size, dtype=np.int64)
+    left_indexer = np.empty(left_size, dtype=np.intp)
+    right_indexer = np.empty(left_size, dtype=np.intp)
 
     if by_t is object:
         hash_table = PyObjectHashTable(right_size)
@@ -747,7 +747,7 @@ def asof_join_forward_on_X_by_Y(asof_t[:] left_values,
 
     cdef:
         Py_ssize_t left_pos, right_pos, left_size, right_size, found_right_pos
-        ndarray[int64_t] left_indexer, right_indexer
+        ndarray[intp_t] left_indexer, right_indexer
         bint has_tolerance = False
         asof_t tolerance_ = 0
         asof_t diff = 0
@@ -762,8 +762,8 @@ def asof_join_forward_on_X_by_Y(asof_t[:] left_values,
     left_size = len(left_values)
     right_size = len(right_values)
 
-    left_indexer = np.empty(left_size, dtype=np.int64)
-    right_indexer = np.empty(left_size, dtype=np.int64)
+    left_indexer = np.empty(left_size, dtype=np.intp)
+    right_indexer = np.empty(left_size, dtype=np.intp)
 
     if by_t is object:
         hash_table = PyObjectHashTable(right_size)
@@ -816,14 +816,14 @@ def asof_join_nearest_on_X_by_Y(asof_t[:] left_values,
 
     cdef:
         Py_ssize_t left_size, right_size, i
-        ndarray[int64_t] left_indexer, right_indexer, bli, bri, fli, fri
+        ndarray[intp_t] left_indexer, right_indexer, bli, bri, fli, fri
         asof_t bdiff, fdiff
 
     left_size = len(left_values)
     right_size = len(right_values)
 
-    left_indexer = np.empty(left_size, dtype=np.int64)
-    right_indexer = np.empty(left_size, dtype=np.int64)
+    left_indexer = np.empty(left_size, dtype=np.intp)
+    right_indexer = np.empty(left_size, dtype=np.intp)
 
     # search both forward and backward
     bli, bri = asof_join_backward_on_X_by_Y(
@@ -867,7 +867,7 @@ def asof_join_backward(asof_t[:] left_values,
 
     cdef:
         Py_ssize_t left_pos, right_pos, left_size, right_size
-        ndarray[int64_t] left_indexer, right_indexer
+        ndarray[intp_t] left_indexer, right_indexer
         bint has_tolerance = False
         asof_t tolerance_ = 0
         asof_t diff = 0
@@ -880,8 +880,8 @@ def asof_join_backward(asof_t[:] left_values,
     left_size = len(left_values)
     right_size = len(right_values)
 
-    left_indexer = np.empty(left_size, dtype=np.int64)
-    right_indexer = np.empty(left_size, dtype=np.int64)
+    left_indexer = np.empty(left_size, dtype=np.intp)
+    right_indexer = np.empty(left_size, dtype=np.intp)
 
     right_pos = 0
     for left_pos in range(left_size):
@@ -920,7 +920,7 @@ def asof_join_forward(asof_t[:] left_values,
 
     cdef:
         Py_ssize_t left_pos, right_pos, left_size, right_size
-        ndarray[int64_t] left_indexer, right_indexer
+        ndarray[intp_t] left_indexer, right_indexer
         bint has_tolerance = False
         asof_t tolerance_ = 0
         asof_t diff = 0
@@ -933,8 +933,8 @@ def asof_join_forward(asof_t[:] left_values,
     left_size = len(left_values)
     right_size = len(right_values)
 
-    left_indexer = np.empty(left_size, dtype=np.int64)
-    right_indexer = np.empty(left_size, dtype=np.int64)
+    left_indexer = np.empty(left_size, dtype=np.intp)
+    right_indexer = np.empty(left_size, dtype=np.intp)
 
     right_pos = right_size - 1
     for left_pos in range(left_size - 1, -1, -1):
@@ -974,14 +974,14 @@ def asof_join_nearest(asof_t[:] left_values,
 
     cdef:
         Py_ssize_t left_size, right_size, i
-        ndarray[int64_t] left_indexer, right_indexer, bli, bri, fli, fri
+        ndarray[intp_t] left_indexer, right_indexer, bli, bri, fli, fri
         asof_t bdiff, fdiff
 
     left_size = len(left_values)
     right_size = len(right_values)
 
-    left_indexer = np.empty(left_size, dtype=np.int64)
-    right_indexer = np.empty(left_size, dtype=np.int64)
+    left_indexer = np.empty(left_size, dtype=np.intp)
+    right_indexer = np.empty(left_size, dtype=np.intp)
 
     # search both forward and backward
     bli, bri = asof_join_backward(left_values, right_values,