PERF: improved clip performance (#16364)

closes #15400
(cherry picked from commit 42e2a87)

Author: jreback

asv_bench/benchmarks/series_methods.py

@@ -111,6 +111,7 @@ def setup(self):
     def time_series_dropna_int64(self):
         self.s.dropna()
 
+
 class series_dropna_datetime(object):
     goal_time = 0.2
 
@@ -120,3 +121,13 @@ def setup(self):
 
     def time_series_dropna_datetime(self):
         self.s.dropna()
+
+
+class series_clip(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = pd.Series(np.random.randn(50))
+
+    def time_series_dropna_datetime(self):
+        self.s.clip(0, 1)

doc/source/whatsnew/v0.20.2.txt

@@ -19,7 +19,7 @@ Highlights include:
 Enhancements
 ~~~~~~~~~~~~
 
-- Unblocked access to additional compression types supported in pytables: 'blosc:blosclz, 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd' (:issue:`14478`) 
+- Unblocked access to additional compression types supported in pytables: 'blosc:blosclz, 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd' (:issue:`14478`)
 
 .. _whatsnew_0202.performance:
 
@@ -30,7 +30,6 @@ Performance Improvements
 - Performance regression fix for MultiIndexes (:issue:`16319`, :issue:`16346`)
 - Improved performance of ``.clip()`` with scalar arguments (:issue:`15400`)
 
-
 .. _whatsnew_0202.bug_fixes:
 
 Bug Fixes

pandas/core/generic.py

@@ -14,6 +14,7 @@
     _ensure_int64,
     needs_i8_conversion,
     is_scalar,
+    is_number,
     is_integer, is_bool,
     is_bool_dtype,
     is_numeric_dtype,
@@ -4104,6 +4105,22 @@ def isnull(self):
     def notnull(self):
         return notnull(self).__finalize__(self)
 
+    def _clip_with_scalar(self, lower, upper):
+
+        if ((lower is not None and np.any(isnull(lower))) or
+                (upper is not None and np.any(isnull(upper)))):
+            raise ValueError("Cannot use an NA value as a clip threshold")
+
+        result = self.values
+        mask = isnull(result)
+        if upper is not None:
+            result = np.where(result >= upper, upper, result)
+        if lower is not None:
+            result = np.where(result <= lower, lower, result)
+        result[mask] = np.nan
+        return self._constructor(
+            result, **self._construct_axes_dict()).__finalize__(self)
+
     def clip(self, lower=None, upper=None, axis=None, *args, **kwargs):
         """
         Trim values at input threshold(s).
@@ -4122,26 +4139,29 @@ def clip(self, lower=None, upper=None, axis=None, *args, **kwargs):
         Examples
         --------
         >>> df
-          0         1
+                  0         1
         0  0.335232 -1.256177
         1 -1.367855  0.746646
         2  0.027753 -1.176076
         3  0.230930 -0.679613
         4  1.261967  0.570967
+
         >>> df.clip(-1.0, 0.5)
                   0         1
         0  0.335232 -1.000000
         1 -1.000000  0.500000
         2  0.027753 -1.000000
         3  0.230930 -0.679613
         4  0.500000  0.500000
+
         >>> t
         0   -0.3
         1   -0.2
         2   -0.1
         3    0.0
         4    0.1
         dtype: float64
+
         >>> df.clip(t, t + 1, axis=0)
                   0         1
         0  0.335232 -0.300000
@@ -4160,6 +4180,11 @@ def clip(self, lower=None, upper=None, axis=None, *args, **kwargs):
             if is_scalar(lower) and is_scalar(upper):
                 lower, upper = min(lower, upper), max(lower, upper)
 
+        # fast-path for scalars
+        if ((lower is None or (is_scalar(lower) and is_number(lower))) and
+                (upper is None or (is_scalar(upper) and is_number(upper)))):
+            return self._clip_with_scalar(lower, upper)
+
         result = self
         if lower is not None:
             result = result.clip_lower(lower, axis)
@@ -4189,6 +4214,9 @@ def clip_upper(self, threshold, axis=None):
         if np.any(isnull(threshold)):
             raise ValueError("Cannot use an NA value as a clip threshold")
 
+        if is_scalar(threshold) and is_number(threshold):
+            return self._clip_with_scalar(None, threshold)
+
         subset = self.le(threshold, axis=axis) | isnull(self)
         return self.where(subset, threshold, axis=axis)
 
@@ -4213,6 +4241,9 @@ def clip_lower(self, threshold, axis=None):
         if np.any(isnull(threshold)):
             raise ValueError("Cannot use an NA value as a clip threshold")
 
+        if is_scalar(threshold) and is_number(threshold):
+            return self._clip_with_scalar(threshold, None)
+
         subset = self.ge(threshold, axis=axis) | isnull(self)
         return self.where(subset, threshold, axis=axis)
 

pandas/tests/series/test_analytics.py

@@ -1011,6 +1011,7 @@ def test_clip_against_series(self):
 
         lower = Series([1.0, 2.0, 3.0])
         upper = Series([1.5, 2.5, 3.5])
+
         assert_series_equal(s.clip(lower, upper), Series([1.0, 2.0, 3.5]))
         assert_series_equal(s.clip(1.5, upper), Series([1.5, 1.5, 3.5]))