BUG/PERF: sparse min/max don't densify (pandas-dev#43527)

Author: mzeitlin11

asv_bench/benchmarks/sparse.py

@@ -166,4 +166,18 @@ def time_division(self, fill_value):
         self.arr1 / self.arr2
 
 
+class MinMax:
+
+    params = (["min", "max"], [0.0, np.nan])
+    param_names = ["func", "fill_value"]
+
+    def setup(self, func, fill_value):
+        N = 1_000_000
+        arr = make_array(N, 1e-5, fill_value, np.float64)
+        self.sp_arr = SparseArray(arr, fill_value=fill_value)
+
+    def time_min_max(self, func, fill_value):
+        getattr(self.sp_arr, func)()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

doc/source/whatsnew/v1.4.0.rst

@@ -302,6 +302,7 @@ Performance improvements
 - Performance improvement in :meth:`Series.sparse.to_coo` (:issue:`42880`)
 - Performance improvement in indexing with a :class:`MultiIndex` indexer on another :class:`MultiIndex` (:issue:43370`)
 - Performance improvement in :meth:`GroupBy.quantile` (:issue:`43469`)
+- :meth:`SparseArray.min` and :meth:`SparseArray.max` no longer require converting to a dense array (:issue:`43526`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -437,6 +438,7 @@ Reshaping
 Sparse
 ^^^^^^
 - Bug in :meth:`DataFrame.sparse.to_coo` raising ``AttributeError`` when column names are not unique (:issue:`29564`)
+- Bug in :meth:`SparseArray.max` and :meth:`SparseArray.min` raising ``ValueError`` for arrays with 0 non-null elements (:issue:`43527`)
 -
 -
 

pandas/core/arrays/sparse/array.py

@@ -1456,23 +1456,69 @@ def mean(self, axis=0, *args, **kwargs):
             nsparse = self.sp_index.ngaps
             return (sp_sum + self.fill_value * nsparse) / (ct + nsparse)
 
-    def max(self, axis=0, *args, **kwargs):
+    def max(self, axis: int = 0, *args, **kwargs) -> Scalar:
+        """
+        Max of non-NA/null values
+
+        Parameters
+        ----------
+        axis : int, default 0
+            Not Used. NumPy compatibility.
+        *args, **kwargs
+            Not Used. NumPy compatibility.
+
+        Returns
+        -------
+        scalar
+        """
         nv.validate_max(args, kwargs)
+        return self._min_max("max")
 
-        # This condition returns a nan if there are no valid values in the array.
-        if self.size > 0 and self._valid_sp_values.size == 0:
-            return self.fill_value
-        else:
-            return np.nanmax(self, axis)
+    def min(self, axis: int = 0, *args, **kwargs) -> Scalar:
+        """
+        Min of non-NA/null values
+
+        Parameters
+        ----------
+        axis : int, default 0
+            Not Used. NumPy compatibility.
+        *args, **kwargs
+            Not Used. NumPy compatibility.
 
-    def min(self, axis=0, *args, **kwargs):
+        Returns
+        -------
+        scalar
+        """
         nv.validate_min(args, kwargs)
+        return self._min_max("min")
+
+    def _min_max(self, kind: Literal["min", "max"]) -> Scalar:
+        """
+        Min/max of non-NA/null values
 
-        # This condition returns a nan if there are no valid values in the array.
-        if self.size > 0 and self._valid_sp_values.size == 0:
+        Parameters
+        ----------
+        kind : {"min", "max"}
+
+        Returns
+        -------
+        scalar
+        """
+        valid_vals = self._valid_sp_values
+        has_nonnull_fill_vals = not self._null_fill_value and self.sp_index.ngaps > 0
+        if len(valid_vals) > 0:
+            sp_min_max = getattr(valid_vals, kind)()
+
+            # If a non-null fill value is currently present, it might be the min/max
+            if has_nonnull_fill_vals:
+                func = max if kind == "max" else min
+                return func(sp_min_max, self.fill_value)
+            else:
+                return sp_min_max
+        elif has_nonnull_fill_vals:
             return self.fill_value
         else:
-            return np.nanmin(self, axis)
+            return na_value_for_dtype(self.dtype.subtype)
 
     # ------------------------------------------------------------------------
     # Ufuncs

pandas/tests/arrays/sparse/test_array.py

@@ -1362,26 +1362,54 @@ def test_drop_duplicates_fill_value():
 
 
 class TestMinMax:
-    plain_data = np.arange(5).astype(float)
-    data_neg = plain_data * (-1)
-    data_NaN = SparseArray(np.array([0, 1, 2, np.nan, 4]))
-    data_all_NaN = SparseArray(np.array([np.nan, np.nan, np.nan, np.nan, np.nan]))
-    data_NA_filled = SparseArray(
-        np.array([np.nan, np.nan, np.nan, np.nan, np.nan]), fill_value=5
-    )
-
     @pytest.mark.parametrize(
         "raw_data,max_expected,min_expected",
         [
-            (plain_data, [4], [0]),
-            (data_neg, [0], [-4]),
-            (data_NaN, [4], [0]),
-            (data_all_NaN, [np.nan], [np.nan]),
-            (data_NA_filled, [5], [5]),
+            (np.arange(5.0), [4], [0]),
+            (-np.arange(5.0), [0], [-4]),
+            (np.array([0, 1, 2, np.nan, 4]), [4], [0]),
+            (np.array([np.nan] * 5), [np.nan], [np.nan]),
+            (np.array([]), [np.nan], [np.nan]),
         ],
     )
-    def test_maxmin(self, raw_data, max_expected, min_expected):
+    def test_nan_fill_value(self, raw_data, max_expected, min_expected):
         max_result = SparseArray(raw_data).max()
         min_result = SparseArray(raw_data).min()
         assert max_result in max_expected
         assert min_result in min_expected
+
+    @pytest.mark.parametrize(
+        "fill_value,max_expected,min_expected",
+        [
+            (100, 100, 0),
+            (-100, 1, -100),
+        ],
+    )
+    def test_fill_value(self, fill_value, max_expected, min_expected):
+        arr = SparseArray(
+            np.array([fill_value, 0, 1]), dtype=SparseDtype("int", fill_value)
+        )
+        max_result = arr.max()
+        assert max_result == max_expected
+
+        min_result = arr.min()
+        assert min_result == min_expected
+
+    @pytest.mark.parametrize("func", ["min", "max"])
+    @pytest.mark.parametrize("data", [np.array([]), np.array([np.nan, np.nan])])
+    @pytest.mark.parametrize(
+        "dtype,expected",
+        [
+            (SparseDtype(np.float64, np.nan), np.nan),
+            (SparseDtype(np.float64, 5.0), np.nan),
+            (SparseDtype("datetime64[ns]", pd.NaT), pd.NaT),
+            (SparseDtype("datetime64[ns]", pd.to_datetime("2018-05-05")), pd.NaT),
+        ],
+    )
+    def test_na_value_if_no_valid_values(self, func, data, dtype, expected):
+        arr = SparseArray(data, dtype=dtype)
+        result = getattr(arr, func)()
+        if expected == pd.NaT:
+            assert result == pd.NaT
+        else:
+            assert np.isnan(result)