PERF: Fix reference leak in read_hdf

asv_bench/benchmarks/io/hdf.py

@@ -128,9 +128,17 @@ def setup(self, format):
         self.df["object"] = tm.makeStringIndex(N)
         self.df.to_hdf(self.fname, "df", format=format)
 
+        # Numeric df
+        self.df1 = self.df.copy()
+        self.df1 = self.df1.reset_index()
+        self.df1.to_hdf(self.fname, "df1", format=format)
+
     def time_read_hdf(self, format):
         read_hdf(self.fname, "df")
 
+    def peakmem_read_hdf(self, format):
+        read_hdf(self.fname, "df")
+
     def time_write_hdf(self, format):
         self.df.to_hdf(self.fname, "df", format=format)
 

doc/source/whatsnew/v2.0.0.rst

@@ -857,6 +857,7 @@ Performance improvements
 - Performance improvement in :func:`to_datetime` when format is given or can be inferred (:issue:`50465`)
 - Performance improvement in :func:`read_csv` when passing :func:`to_datetime` lambda-function to ``date_parser`` and inputs have mixed timezone offsetes (:issue:`35296`)
 - Performance improvement in :meth:`.SeriesGroupBy.value_counts` with categorical dtype (:issue:`46202`)
+- Fixed a reference leak in :func:`read_hdf` (:issue:`37441`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.bug_fixes:

pandas/io/pytables.py

@@ -2057,7 +2057,9 @@ def convert(
 
         # values is a recarray
         if values.dtype.fields is not None:
-            values = values[self.cname]
+            # Copy, otherwise values will be a view
+            # preventing the original recarry from being free'ed
+            values = values[self.cname].copy()
 
         val_kind = _ensure_decoded(self.kind)
         values = _maybe_convert(values, val_kind, encoding, errors)

pandas/tests/io/pytables/test_read.py

@@ -214,6 +214,20 @@ def test_read_hdf_open_store(tmp_path, setup_path):
         assert store.is_open
 
 
+def test_read_hdf_index_not_view(tmp_path, setup_path):
+    # GH 37441
+    # Ensure that the index of the DataFrame is not a view
+    # into the original recarray that pytables reads in
+    df = DataFrame(np.random.rand(4, 5), index=[0, 1, 2, 3], columns=list("ABCDE"))
+
+    path = tmp_path / setup_path
+    df.to_hdf(path, "df", mode="w", format="table")
+
+    df2 = read_hdf(path, "df")
+    assert df2.index._data.base is None
+    tm.assert_frame_equal(df, df2)
+
+
 def test_read_hdf_iterator(tmp_path, setup_path):
     df = DataFrame(np.random.rand(4, 5), index=list("abcd"), columns=list("ABCDE"))
     df.index.name = "letters"