diff --git a/asv_bench/benchmarks/io/hdf.py b/asv_bench/benchmarks/io/hdf.py index 12bc65f9e7bf5..e44a59114b30d 100644 --- a/asv_bench/benchmarks/io/hdf.py +++ b/asv_bench/benchmarks/io/hdf.py @@ -128,9 +128,17 @@ def setup(self, format): self.df["object"] = tm.makeStringIndex(N) self.df.to_hdf(self.fname, "df", format=format) + # Numeric df + self.df1 = self.df.copy() + self.df1 = self.df1.reset_index() + self.df1.to_hdf(self.fname, "df1", format=format) + def time_read_hdf(self, format): read_hdf(self.fname, "df") + def peakmem_read_hdf(self, format): + read_hdf(self.fname, "df") + def time_write_hdf(self, format): self.df.to_hdf(self.fname, "df", format=format) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 1c99ba0b8e412..2e1be0623441a 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -857,6 +857,7 @@ Performance improvements - Performance improvement in :func:`to_datetime` when format is given or can be inferred (:issue:`50465`) - Performance improvement in :func:`read_csv` when passing :func:`to_datetime` lambda-function to ``date_parser`` and inputs have mixed timezone offsetes (:issue:`35296`) - Performance improvement in :meth:`.SeriesGroupBy.value_counts` with categorical dtype (:issue:`46202`) +- Fixed a reference leak in :func:`read_hdf` (:issue:`37441`) .. --------------------------------------------------------------------------- .. _whatsnew_200.bug_fixes: diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 8c0d5c3da385c..eedcf46fd063e 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -2057,7 +2057,9 @@ def convert( # values is a recarray if values.dtype.fields is not None: - values = values[self.cname] + # Copy, otherwise values will be a view + # preventing the original recarry from being free'ed + values = values[self.cname].copy() val_kind = _ensure_decoded(self.kind) values = _maybe_convert(values, val_kind, encoding, errors) diff --git a/pandas/tests/io/pytables/test_read.py b/pandas/tests/io/pytables/test_read.py index 6d92c15f1ea10..cb0c1ce35c7c7 100644 --- a/pandas/tests/io/pytables/test_read.py +++ b/pandas/tests/io/pytables/test_read.py @@ -214,6 +214,20 @@ def test_read_hdf_open_store(tmp_path, setup_path): assert store.is_open +def test_read_hdf_index_not_view(tmp_path, setup_path): + # GH 37441 + # Ensure that the index of the DataFrame is not a view + # into the original recarray that pytables reads in + df = DataFrame(np.random.rand(4, 5), index=[0, 1, 2, 3], columns=list("ABCDE")) + + path = tmp_path / setup_path + df.to_hdf(path, "df", mode="w", format="table") + + df2 = read_hdf(path, "df") + assert df2.index._data.base is None + tm.assert_frame_equal(df, df2) + + def test_read_hdf_iterator(tmp_path, setup_path): df = DataFrame(np.random.rand(4, 5), index=list("abcd"), columns=list("ABCDE")) df.index.name = "letters"