From f336549399be3d4b5aa20d4ecaa5ddfaede46ac2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 19 Apr 2023 16:17:57 +0100 Subject: [PATCH 1/6] add IndexError to the types of possible read_hdf errors fixes #52781 --- pandas/io/pytables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 2a522ef6b5171..fc87321b97d54 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -455,7 +455,7 @@ def read_hdf( chunksize=chunksize, auto_close=auto_close, ) - except (ValueError, TypeError, KeyError): + except (ValueError, TypeError, KeyError, IndexError): if not isinstance(path_or_buf, HDFStore): # if there is an error, close the store if we opened it. with suppress(AttributeError): From a504f4f4eb4455766dea534b41a6f0fd420e6917 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 19 Apr 2023 17:01:17 +0100 Subject: [PATCH 2/6] test that IndexError closes hdf files --- pandas/tests/io/pytables/test_read.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pandas/tests/io/pytables/test_read.py b/pandas/tests/io/pytables/test_read.py index 2f2190a352045..2fe8e2e28a6f8 100644 --- a/pandas/tests/io/pytables/test_read.py +++ b/pandas/tests/io/pytables/test_read.py @@ -42,6 +42,20 @@ def test_read_missing_key_close_store(tmp_path, setup_path): df.to_hdf(path, "k2") +def test_read_index_error_close_store(tmp_path, setup_path): + # GH 25766 + path = tmp_path / setup_path + df = pd.DataFrame({"A": [], "B": []}, index=[]) + df.to_hdf(path, "k1") + + with pytest.raises(IndexError, match=r"list index out of range"): + read_hdf(path, "k1", stop=0) + + # smoke test to test that file is properly closed after + # read with IndexError before another write + df.to_hdf(path, "k1") + + def test_read_missing_key_opened_store(tmp_path, setup_path): # GH 28699 path = tmp_path / setup_path From a70fd019e82a38b61894a3e46461370d68d953b5 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 19 Apr 2023 17:04:50 +0100 Subject: [PATCH 3/6] coalesce KeyError and IndexError into LookupError --- pandas/io/pytables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index fc87321b97d54..8de1aaacaf400 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -455,7 +455,7 @@ def read_hdf( chunksize=chunksize, auto_close=auto_close, ) - except (ValueError, TypeError, KeyError, IndexError): + except (ValueError, TypeError, LookupError): if not isinstance(path_or_buf, HDFStore): # if there is an error, close the store if we opened it. with suppress(AttributeError): From 1f170fb05d3528a4cdf74c439434cbc737229c85 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 19 Apr 2023 17:07:09 +0100 Subject: [PATCH 4/6] add news entry --- doc/source/whatsnew/v2.1.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 22a2931519ffd..5cbbb31ccd138 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -350,6 +350,7 @@ I/O - :meth:`DataFrame.to_sql` now raising ``ValueError`` when the name param is left empty while using SQLAlchemy to connect (:issue:`52675`) - Bug in :func:`read_html`, style elements were read into DataFrames (:issue:`52197`) - Bug in :func:`read_html`, tail texts were removed together with elements containing ``display:none`` style (:issue:`51629`) +- Bug in :func:`read_hdf` not properly closing store after a ``IndexError`` is raised (:issue:`52781`) - Period From efc0dcccaab0bd2024d66c627510f47fb339bc6c Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 19 Apr 2023 17:39:56 +0100 Subject: [PATCH 5/6] avoid pd. prefix --- pandas/tests/io/pytables/test_read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/pytables/test_read.py b/pandas/tests/io/pytables/test_read.py index 2fe8e2e28a6f8..61dabf15653f0 100644 --- a/pandas/tests/io/pytables/test_read.py +++ b/pandas/tests/io/pytables/test_read.py @@ -45,7 +45,7 @@ def test_read_missing_key_close_store(tmp_path, setup_path): def test_read_index_error_close_store(tmp_path, setup_path): # GH 25766 path = tmp_path / setup_path - df = pd.DataFrame({"A": [], "B": []}, index=[]) + df = DataFrame({"A": [], "B": []}, index=[]) df.to_hdf(path, "k1") with pytest.raises(IndexError, match=r"list index out of range"): From 8f9d9649259cb4d183e69461677a71437b0954c1 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 19 Apr 2023 17:40:10 +0100 Subject: [PATCH 6/6] sort changelog --- doc/source/whatsnew/v2.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 5cbbb31ccd138..4ff0a7ef0022e 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -348,9 +348,9 @@ I/O ^^^ - :meth:`DataFrame.to_orc` now raising ``ValueError`` when non-default :class:`Index` is given (:issue:`51828`) - :meth:`DataFrame.to_sql` now raising ``ValueError`` when the name param is left empty while using SQLAlchemy to connect (:issue:`52675`) +- Bug in :func:`read_hdf` not properly closing store after a ``IndexError`` is raised (:issue:`52781`) - Bug in :func:`read_html`, style elements were read into DataFrames (:issue:`52197`) - Bug in :func:`read_html`, tail texts were removed together with elements containing ``display:none`` style (:issue:`51629`) -- Bug in :func:`read_hdf` not properly closing store after a ``IndexError`` is raised (:issue:`52781`) - Period