PERF: quantile now operates per block boosting perf / fix quantile with nan

closes #11623
closes #13098

Author: Jeff Reback <[email protected]>

Closes #13122 from jreback/quantile and squashes the following commits:

aad72cb [Jeff Reback] PERF: quantile now operates per block boosting perf REGR: series quantile with nan

Author: jreback

asv_bench/benchmarks/frame_methods.py

@@ -423,7 +423,7 @@ class frame_get_dtype_counts(object):
     goal_time = 0.2
 
     def setup(self):
-        self.df = pandas.DataFrame(np.random.randn(10, 10000))
+        self.df = DataFrame(np.random.randn(10, 10000))
 
     def time_frame_get_dtype_counts(self):
         self.df.get_dtype_counts()
@@ -985,3 +985,14 @@ def setup(self):
 
     def time_series_string_vector_slice(self):
         self.s.str[:5]
+
+
+class frame_quantile_axis1(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.df = DataFrame(np.random.randn(1000, 3),
+                            columns=list('ABC'))
+
+    def time_frame_quantile_axis1(self):
+        self.df.quantile([0.1, 0.5], axis=1)

codecov.yml

@@ -9,4 +9,5 @@ coverage:
         branches: null
     changes:
       default:
-        branches: null
+        branches:
+          - master

doc/source/whatsnew/v0.18.1.txt

@@ -563,7 +563,6 @@ Performance Improvements
 - Improved speed of SAS reader (:issue:`12656`, :issue:`12961`)
 - Performance improvements in ``.groupby(..).cumcount()`` (:issue:`11039`)
 - Improved memory usage in ``pd.read_csv()`` when using ``skiprows=an_integer`` (:issue:`13005`)
-
 - Improved performance of ``DataFrame.to_sql`` when checking case sensitivity for tables. Now only checks if table has been created correctly when table name is not lower case. (:issue:`12876`)
 - Improved performance of ``Period`` construction and time series plotting (:issue:`12903`, :issue:`11831`).
 - Improved performance of ``.str.encode()`` and ``.str.decode()`` methods (:issue:`13008`)

doc/source/whatsnew/v0.18.2.txt

@@ -97,6 +97,9 @@ Performance Improvements
 
 - Improved performance of sparse ``IntIndex.intersect`` (:issue:`13082`)
 - Improved performance of sparse arithmetic with ``BlockIndex`` when the number of blocks are large, though recommended to use ``IntIndex`` in such cases (:issue:`13082`)
+- increased performance of ``DataFrame.quantile()`` as it now operates per-block (:issue:`11623`)
+
+
 
 
 
@@ -110,6 +113,7 @@ Bug Fixes
 
 
 
+- Regression in ``Series.quantile`` with nans (also shows up in ``.median()`` and ``.describe()``); furthermore now names the ``Series`` with the quantile (:issue:`13098`, :issue:`13146`)
 
 
 
@@ -135,15 +139,3 @@ Bug Fixes
 - Bug in ``NaT`` - ``Period`` raises ``AttributeError`` (:issue:`13071`)
 - Bug in ``Period`` addition raises ``TypeError`` if ``Period`` is on right hand side (:issue:`13069`)
 - Bug in ``pd.set_eng_float_format()`` that would prevent NaN's from formatting (:issue:`11981`)
-
-
-
-
-
-
-
-
-
-
-
-

pandas/core/frame.py

@@ -4989,31 +4989,27 @@ def quantile(self, q=0.5, axis=0, numeric_only=True,
         0.5  2.5  55.0
         """
         self._check_percentile(q)
-        if not com.is_list_like(q):
-            q = [q]
-            squeeze = True
-        else:
-            squeeze = False
 
         data = self._get_numeric_data() if numeric_only else self
         axis = self._get_axis_number(axis)
+        is_transposed = axis == 1
 
-        def _quantile(series):
-            res = series.quantile(q, interpolation=interpolation)
-            return series.name, res
-
-        if axis == 1:
+        if is_transposed:
             data = data.T
 
-        # unable to use DataFrame.apply, becasuse data may be empty
-        result = dict(_quantile(s) for (_, s) in data.iteritems())
-        result = self._constructor(result, columns=data.columns)
-        if squeeze:
-            if result.shape == (1, 1):
-                result = result.T.iloc[:, 0]  # don't want scalar
-            else:
-                result = result.T.squeeze()
-            result.name = None  # For groupby, so it can set an index name
+        result = data._data.quantile(qs=q,
+                                     axis=1,
+                                     interpolation=interpolation,
+                                     transposed=is_transposed)
+
+        if result.ndim == 2:
+            result = self._constructor(result)
+        else:
+            result = self._constructor_sliced(result, name=q)
+
+        if is_transposed:
+            result = result.T
+
         return result
 
     def to_timestamp(self, freq=None, how='start', axis=0, copy=True):