Merge pull request #153 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

.pre-commit-config.yaml

@@ -47,11 +47,11 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.7.0
+    rev: 5.8.0
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.10.0
+    rev: v2.11.0
     hooks:
     -   id: pyupgrade
         args: [--py37-plus, --keep-runtime-typing]

asv_bench/benchmarks/io/style.py

@@ -3,7 +3,7 @@
 from pandas import DataFrame
 
 
-class RenderApply:
+class Render:
 
     params = [[12, 24, 36], [12, 120]]
     param_names = ["cols", "rows"]
@@ -14,15 +14,21 @@ def setup(self, cols, rows):
             columns=[f"float_{i+1}" for i in range(cols)],
             index=[f"row_{i+1}" for i in range(rows)],
         )
-        self._style_apply()
 
-    def time_render(self, cols, rows):
+    def time_apply_render(self, cols, rows):
+        self._style_apply()
         self.st.render()
 
-    def peakmem_apply(self, cols, rows):
+    def peakmem_apply_render(self, cols, rows):
         self._style_apply()
+        self.st.render()
 
-    def peakmem_render(self, cols, rows):
+    def time_classes_render(self, cols, rows):
+        self._style_classes()
+        self.st.render()
+
+    def peakmem_classes_render(self, cols, rows):
+        self._style_classes()
         self.st.render()
 
     def _style_apply(self):
@@ -32,3 +38,8 @@ def _apply_func(s):
             ]
 
         self.st = self.df.style.apply(_apply_func, axis=1)
+
+    def _style_classes(self):
+        classes = self.df.applymap(lambda v: ("cls-1" if v > 0 else ""))
+        classes.index, classes.columns = self.df.index, self.df.columns
+        self.st = self.df.style.set_td_classes(classes)

ci/code_checks.sh

@@ -118,7 +118,7 @@ import sys
 import pandas
 
 blocklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
-             'lxml', 'matplotlib', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
+             'lxml', 'matplotlib', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
              'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
 
 # GH#28227 for some of these check for top-level modules, while others are

doc/make.py

@@ -54,7 +54,6 @@ def __init__(
 
         if single_doc:
             single_doc = self._process_single_doc(single_doc)
-            include_api = False
             os.environ["SPHINX_PATTERN"] = single_doc
         elif not include_api:
             os.environ["SPHINX_PATTERN"] = "-api"

doc/source/whatsnew/v1.2.4.rst

@@ -16,6 +16,7 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
 - Fixed regression in :meth:`DataFrame.sum` when ``min_count`` greater than the :class:`DataFrame` shape was passed resulted in a ``ValueError`` (:issue:`39738`)
+- Fixed regression in :meth:`DataFrame.to_json` raising ``AttributeError`` when run on PyPy (:issue:`39837`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.3.0.rst

@@ -128,6 +128,7 @@ Other enhancements
 - :meth:`.Rolling.sum`, :meth:`.Expanding.sum`, :meth:`.Rolling.mean`, :meth:`.Expanding.mean`, :meth:`.Rolling.median`, :meth:`.Expanding.median`, :meth:`.Rolling.max`, :meth:`.Expanding.max`, :meth:`.Rolling.min`, and :meth:`.Expanding.min` now support ``Numba`` execution with the ``engine`` keyword (:issue:`38895`)
 - :meth:`DataFrame.apply` can now accept NumPy unary operators as strings, e.g. ``df.apply("sqrt")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
 - :meth:`DataFrame.apply` can now accept non-callable DataFrame properties as strings, e.g. ``df.apply("size")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
+- :meth:`DataFrame.applymap` can now accept kwargs to pass on to func (:issue:`39987`)
 - Disallow :class:`DataFrame` indexer for ``iloc`` for :meth:`Series.__getitem__` and :meth:`DataFrame.__getitem__`, (:issue:`39004`)
 - :meth:`Series.apply` can now accept list-like or dictionary-like arguments that aren't lists or dictionaries, e.g. ``ser.apply(np.array(["sum", "mean"]))``, which was already the case for :meth:`DataFrame.apply` (:issue:`39140`)
 - :meth:`DataFrame.plot.scatter` can now accept a categorical column as the argument to ``c`` (:issue:`12380`, :issue:`31357`)

pandas/_libs/algos.pxd

@@ -1,21 +1,4 @@
 from pandas._libs.util cimport numeric
 
 
-cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil:
-    cdef:
-        numeric t
-
-    # cython doesn't allow pointer dereference so use array syntax
-    t = a[0]
-    a[0] = b[0]
-    b[0] = t
-    return 0
-
-
-cdef enum TiebreakEnumType:
-    TIEBREAK_AVERAGE
-    TIEBREAK_MIN,
-    TIEBREAK_MAX
-    TIEBREAK_FIRST
-    TIEBREAK_FIRST_DESCENDING
-    TIEBREAK_DENSE
+cdef numeric kth_smallest_c(numeric* arr, Py_ssize_t k, Py_ssize_t n) nogil

pandas/_libs/algos.pyx

@@ -64,6 +64,14 @@ cdef:
     float64_t NaN = <float64_t>np.NaN
     int64_t NPY_NAT = get_nat()
 
+cdef enum TiebreakEnumType:
+    TIEBREAK_AVERAGE
+    TIEBREAK_MIN,
+    TIEBREAK_MAX
+    TIEBREAK_FIRST
+    TIEBREAK_FIRST_DESCENDING
+    TIEBREAK_DENSE
+
 tiebreakers = {
     "average": TIEBREAK_AVERAGE,
     "min": TIEBREAK_MIN,
@@ -237,34 +245,75 @@ def groupsort_indexer(const int64_t[:] index, Py_ssize_t ngroups):
     return indexer, counts
 
 
+cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil:
+    cdef:
+        numeric t
+
+    # cython doesn't allow pointer dereference so use array syntax
+    t = a[0]
+    a[0] = b[0]
+    b[0] = t
+    return 0
+
+
+cdef inline numeric kth_smallest_c(numeric* arr, Py_ssize_t k, Py_ssize_t n) nogil:
+    """
+    See kth_smallest.__doc__. The additional parameter n specifies the maximum
+    number of elements considered in arr, needed for compatibility with usage
+    in groupby.pyx
+    """
+    cdef:
+        Py_ssize_t i, j, l, m
+        numeric x
+
+    l = 0
+    m = n - 1
+
+    while l < m:
+        x = arr[k]
+        i = l
+        j = m
+
+        while 1:
+            while arr[i] < x: i += 1
+            while x < arr[j]: j -= 1
+            if i <= j:
+                swap(&arr[i], &arr[j])
+                i += 1; j -= 1
+
+            if i > j: break
+
+        if j < k: l = i
+        if k < i: m = j
+    return arr[k]
+
+
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def kth_smallest(numeric[:] a, Py_ssize_t k) -> numeric:
+def kth_smallest(numeric[::1] arr, Py_ssize_t k) -> numeric:
+    """
+    Compute the kth smallest value in arr. Note that the input
+    array will be modified.
+
+    Parameters
+    ----------
+    arr : numeric[::1]
+        Array to compute the kth smallest value for, must be
+        contiguous
+    k : Py_ssize_t
+
+    Returns
+    -------
+    numeric
+        The kth smallest value in arr
+    """
     cdef:
-        Py_ssize_t i, j, l, m, n = a.shape[0]
-        numeric x
+        numeric result
 
     with nogil:
-        l = 0
-        m = n - 1
-
-        while l < m:
-            x = a[k]
-            i = l
-            j = m
-
-            while 1:
-                while a[i] < x: i += 1
-                while x < a[j]: j -= 1
-                if i <= j:
-                    swap(&a[i], &a[j])
-                    i += 1; j -= 1
-
-                if i > j: break
-
-            if j < k: l = i
-            if k < i: m = j
-    return a[k]
+        result = kth_smallest_c(&arr[0], k, arr.shape[0])
+
+    return result
 
 
 # ----------------------------------------------------------------------

pandas/_libs/groupby.pyx

@@ -30,7 +30,7 @@ from numpy.math cimport NAN
 
 cnp.import_array()
 
-from pandas._libs.algos cimport swap
+from pandas._libs.algos cimport kth_smallest_c
 from pandas._libs.util cimport (
     get_nat,
     numeric,
@@ -88,7 +88,7 @@ cdef inline float64_t median_linear(float64_t* a, int n) nogil:
         n -= na_count
 
     if n % 2:
-        result = kth_smallest_c( a, n // 2, n)
+        result = kth_smallest_c(a, n // 2, n)
     else:
         result = (kth_smallest_c(a, n // 2, n) +
                   kth_smallest_c(a, n // 2 - 1, n)) / 2
@@ -99,35 +99,6 @@ cdef inline float64_t median_linear(float64_t* a, int n) nogil:
     return result
 
 
-# TODO: Is this redundant with algos.kth_smallest
-cdef inline float64_t kth_smallest_c(float64_t* a,
-                                     Py_ssize_t k,
-                                     Py_ssize_t n) nogil:
-    cdef:
-        Py_ssize_t i, j, l, m
-        float64_t x, t
-
-    l = 0
-    m = n - 1
-    while l < m:
-        x = a[k]
-        i = l
-        j = m
-
-        while 1:
-            while a[i] < x: i += 1
-            while x < a[j]: j -= 1
-            if i <= j:
-                swap(&a[i], &a[j])
-                i += 1; j -= 1
-
-            if i > j: break
-
-        if j < k: l = i
-        if k < i: m = j
-    return a[k]
-
-
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_median_float64(ndarray[float64_t, ndim=2] out,

pandas/_libs/src/ujson/python/objToJSON.c

@@ -272,18 +272,6 @@ static PyObject *get_sub_attr(PyObject *obj, char *attr, char *subAttr) {
     return ret;
 }
 
-static int is_simple_frame(PyObject *obj) {
-    PyObject *check = get_sub_attr(obj, "_mgr", "is_mixed_type");
-    int ret = (check == Py_False);
-
-    if (!check) {
-        return 0;
-    }
-
-    Py_DECREF(check);
-    return ret;
-}
-
 static Py_ssize_t get_attr_length(PyObject *obj, char *attr) {
     PyObject *tmp = PyObject_GetAttrString(obj, attr);
     Py_ssize_t ret;
@@ -301,6 +289,17 @@ static Py_ssize_t get_attr_length(PyObject *obj, char *attr) {
     return ret;
 }
 
+static int is_simple_frame(PyObject *obj) {
+    PyObject *mgr = PyObject_GetAttrString(obj, "_mgr");
+    if (!mgr) {
+        return 0;
+    }
+    int ret = (get_attr_length(mgr, "blocks") <= 1);
+
+    Py_DECREF(mgr);
+    return ret;
+}
+
 static npy_int64 get_long_attr(PyObject *o, const char *attr) {
     npy_int64 long_val;
     PyObject *value = PyObject_GetAttrString(o, attr);