BUG: Fix getsizeof when using Series(obj) and taking into account GC corrections (#52112)

* BUG: Fix getsizeof when using Series(obj) and taking into account GC corrections

Calling __sizeof__ to compute the size of an object produces an error, and in general, this method does not take Garbage Collector corrections into account.

* BUG: Fix getsizeof when using Series(obj) and taking into account GC corrections

Calling __sizeof__ to compute the size of an object produces an error, and in general, this method does not take Garbage Collector corrections into account.

* Removing unnecessary tests that depend on OS bits

* TST: Included python data types

* TST: Added more objects Series with objects for Memory test

* TST: Added more objects Series with objects for Memory test

* CLN: Deleted unnecesary tests

* TYP: Renamed _object_series to _type_objects_series

Author: balexandermunoz

doc/source/whatsnew/v2.1.0.rst

@@ -257,6 +257,7 @@ Styler
 Other
 ^^^^^
 - Bug in :func:`assert_almost_equal` now throwing assertion error for two unequal sets (:issue:`51727`)
+- Bug in :meth:`Series.memory_usage` when ``deep=True`` throw an error with Series of objects and the returned value is incorrect, as it does not take into account GC corrections (:issue:`51858`)
 
 .. ***DO NOT USE THIS SECTION***
 

pandas/_libs/lib.pyx

@@ -1,6 +1,7 @@
 from collections import abc
 from decimal import Decimal
 from enum import Enum
+from sys import getsizeof
 from typing import (
     Literal,
     _GenericAlias,
@@ -159,7 +160,7 @@ def memory_usage_of_objects(arr: object[:]) -> int64_t:
 
     n = len(arr)
     for i in range(n):
-        size += arr[i].__sizeof__()
+        size += getsizeof(arr[i])
     return size
 
 

pandas/_testing/__init__.py

@@ -177,6 +177,23 @@
     np.uint32,
 ]
 
+PYTHON_DATA_TYPES = [
+    str,
+    int,
+    float,
+    complex,
+    list,
+    tuple,
+    range,
+    dict,
+    set,
+    frozenset,
+    bool,
+    bytes,
+    bytearray,
+    memoryview,
+]
+
 ENDIAN = {"little": "<", "big": ">"}[byteorder]
 
 NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA, Decimal("NaN")]

pandas/conftest.py

@@ -760,6 +760,29 @@ def index_or_series_obj(request):
     return _index_or_series_objs[request.param].copy(deep=True)
 
 
+_typ_objects_series = {
+    f"{dtype.__name__}-series": Series(dtype) for dtype in tm.PYTHON_DATA_TYPES
+}
+
+
+_index_or_series_memory_objs = {
+    **indices_dict,
+    **_series,
+    **_narrow_series,
+    **_typ_objects_series,
+}
+
+
+@pytest.fixture(params=_index_or_series_memory_objs.keys())
+def index_or_series_memory_obj(request):
+    """
+    Fixture for tests on indexes, series, series with a narrow dtype and
+    series with empty objects type
+    copy to avoid mutation, e.g. setting .name
+    """
+    return _index_or_series_memory_objs[request.param].copy(deep=True)
+
+
 # ----------------------------------------------------------------
 # DataFrames
 # ----------------------------------------------------------------

pandas/tests/base/test_misc.py

@@ -82,8 +82,8 @@ def test_ndarray_compat_properties(index_or_series_obj):
 
 
 @pytest.mark.skipif(PYPY, reason="not relevant for PyPy")
-def test_memory_usage(index_or_series_obj):
-    obj = index_or_series_obj
+def test_memory_usage(index_or_series_memory_obj):
+    obj = index_or_series_memory_obj
     # Clear index caches so that len(obj) == 0 report 0 memory usage
     if isinstance(obj, Series):
         is_ser = True