BUG: don't assume series is length > 0

closes #19368

Author: Matthew Kirk <[email protected]>

Closes #19438 from hexgnu/segfault_memory_usage and squashes the following commits:

f9433d8 [Matthew Kirk] Use shared docstring and get rid of if condition
4ead141 [Matthew Kirk] Move whatsnew doc to Sparse
ae9f74d [Matthew Kirk] Revert base.py
cdd4141 [Matthew Kirk] Fix linting error
93a0c3d [Matthew Kirk] Merge remote-tracking branch 'upstream/master' into segfault_memory_usage
207bc74 [Matthew Kirk] Define memory_usage on SparseArray
21ae147 [Matthew Kirk] FIX: revert change to lib.pyx
3f52a44 [Matthew Kirk] Ah ha I think I got it
5e59e9c [Matthew Kirk] Use range over 0 <= for loops
e251587 [Matthew Kirk] Fix failing test with indexing
27df317 [Matthew Kirk] Merge remote-tracking branch 'upstream/master' into segfault_memory_usage
7fdd03e [Matthew Kirk] Take out comment and use product
6bd6ddd [Matthew Kirk] BUG: don't assume series is length > 0

Author: hexgnu

doc/source/whatsnew/v0.23.0.txt

@@ -606,7 +606,7 @@ Sparse
 
 - Bug in which creating a ``SparseDataFrame`` from a dense ``Series`` or an unsupported type raised an uncontrolled exception (:issue:`19374`)
 - Bug in :class:`SparseDataFrame.to_csv` causing exception (:issue:`19384`)
--
+- Bug in :class:`SparseSeries.memory_usage` which caused segfault by accessing non sparse elements (:issue:`19368`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/base.py

@@ -1048,7 +1048,7 @@ def is_monotonic_decreasing(self):
 
     def memory_usage(self, deep=False):
         """
-        Memory usage of my values
+        Memory usage of the values
 
         Parameters
         ----------

pandas/core/sparse/array.py

@@ -8,10 +8,10 @@
 import warnings
 
 import pandas as pd
-from pandas.core.base import PandasObject
+from pandas.core.base import PandasObject, IndexOpsMixin
 
 from pandas import compat
-from pandas.compat import range
+from pandas.compat import range, PYPY
 from pandas.compat.numpy import function as nv
 
 from pandas.core.dtypes.generic import ABCSparseSeries
@@ -30,6 +30,7 @@
 from pandas.core.dtypes.missing import isna, notna, na_value_for_dtype
 
 import pandas._libs.sparse as splib
+import pandas._libs.lib as lib
 from pandas._libs.sparse import SparseIndex, BlockIndex, IntIndex
 from pandas._libs import index as libindex
 import pandas.core.algorithms as algos
@@ -238,6 +239,17 @@ def kind(self):
         elif isinstance(self.sp_index, IntIndex):
             return 'integer'
 
+    @Appender(IndexOpsMixin.memory_usage.__doc__)
+    def memory_usage(self, deep=False):
+        values = self.sp_values
+
+        v = values.nbytes
+
+        if deep and is_object_dtype(self) and not PYPY:
+            v += lib.memory_usage_of_objects(values)
+
+        return v
+
     def __array_wrap__(self, out_arr, context=None):
         """
         NumPy calls this method when ufunc is applied

pandas/tests/sparse/series/test_series.py

@@ -23,6 +23,8 @@
 from pandas.core.sparse.api import SparseSeries
 from pandas.tests.series.test_api import SharedWithSparse
 
+from itertools import product
+
 
 def _test_data1():
     # nan-based
@@ -971,6 +973,17 @@ def test_combine_first(self):
         tm.assert_sp_series_equal(result, result2)
         tm.assert_sp_series_equal(result, expected)
 
+    @pytest.mark.parametrize('deep,fill_values', [([True, False],
+                                                   [0, 1, np.nan, None])])
+    def test_memory_usage_deep(self, deep, fill_values):
+        for deep, fill_value in product(deep, fill_values):
+            sparse_series = SparseSeries(fill_values, fill_value=fill_value)
+            dense_series = Series(fill_values)
+            sparse_usage = sparse_series.memory_usage(deep=deep)
+            dense_usage = dense_series.memory_usage(deep=deep)
+
+            assert sparse_usage < dense_usage
+
 
 class TestSparseHandlingMultiIndexes(object):