From 197782c974e11383854360edb7da645401d8730f Mon Sep 17 00:00:00 2001 From: Laurent Repiton Date: Wed, 9 Jun 2021 16:10:24 +0200 Subject: [PATCH 1/3] BUG: to_hdf append string column to incompatible column --- doc/source/whatsnew/v1.3.0.rst | 1 + pandas/io/pytables.py | 2 +- pandas/tests/io/pytables/test_append.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 849b9d45da5ad..546beba8ce5f1 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -1064,6 +1064,7 @@ I/O - Bug in the conversion from PyArrow to pandas (e.g. for reading Parquet) with nullable dtypes and a PyArrow array whose data buffer size is not a multiple of the dtype size (:issue:`40896`) - Bug in :func:`read_excel` would raise an error when pandas could not determine the file type, even when user specified the ``engine`` argument (:issue:`41225`) - Bug in :func:`read_clipboard` copying from an excel file shifts values into the wrong column if there are null values in first column (:issue:`41108`) +- Bug in :meth:`DataFrame.to_hdf` and :meth:`Series.to_hdf` raising a ``TypeError`` when trying to append a string column to an incompatible column Period ^^^^^^ diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 27c30aa4c10ad..bcbd57fc5dd44 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -4988,7 +4988,7 @@ def _maybe_convert_for_string_atom( # check for column in the values conflicts if existing_col is not None: eci = existing_col.validate_col(itemsize) - if eci > itemsize: + if eci is not None and eci > itemsize: itemsize = eci data_converted = data_converted.astype(f"|S{itemsize}", copy=False) diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index 2569eb0c9e786..20f0f0d461061 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -774,6 +774,20 @@ def test_append_raise(setup_path): with pytest.raises(ValueError, match=msg): store.append("df", df) + # incompatible type + _maybe_remove(store, "df") + df["foo"] = Timestamp("20130101") + store.append("df", df) + df["foo"] = "bar" + msg = re.escape( + "invalid combination of [values_axes] on appending data " + "[name->values_block_1,cname->values_block_1,dtype->bytes24,kind->string,shape->(1, 30)] " + "vs current table " + "[name->values_block_1,cname->values_block_1,dtype->datetime64,kind->datetime64,shape->None]" + ) + with pytest.raises(ValueError, match=msg): + store.append("df", df) + def test_append_with_timedelta(setup_path): # GH 3577 From ed5143acf22707e12e442b78955c3ed1516eec27 Mon Sep 17 00:00:00 2001 From: Laurent Repiton Date: Wed, 9 Jun 2021 16:30:19 +0200 Subject: [PATCH 2/3] Wrapping long lines for PEP 8 --- pandas/tests/io/pytables/test_append.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index 20f0f0d461061..934c851705e9c 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -781,9 +781,11 @@ def test_append_raise(setup_path): df["foo"] = "bar" msg = re.escape( "invalid combination of [values_axes] on appending data " - "[name->values_block_1,cname->values_block_1,dtype->bytes24,kind->string,shape->(1, 30)] " + "[name->values_block_1,cname->values_block_1," + "dtype->bytes24,kind->string,shape->(1, 30)] " "vs current table " - "[name->values_block_1,cname->values_block_1,dtype->datetime64,kind->datetime64,shape->None]" + "[name->values_block_1,cname->values_block_1," + "dtype->datetime64,kind->datetime64,shape->None]" ) with pytest.raises(ValueError, match=msg): store.append("df", df) From 0641bb1e8896ef8823eb96dd8040741bcdd6a9d6 Mon Sep 17 00:00:00 2001 From: Laurent Repiton Date: Mon, 14 Jun 2021 10:17:20 +0200 Subject: [PATCH 3/3] Adding PR number in comments --- doc/source/whatsnew/v1.3.0.rst | 2 +- pandas/tests/io/pytables/test_append.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 546beba8ce5f1..b57b771597edb 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -1064,7 +1064,7 @@ I/O - Bug in the conversion from PyArrow to pandas (e.g. for reading Parquet) with nullable dtypes and a PyArrow array whose data buffer size is not a multiple of the dtype size (:issue:`40896`) - Bug in :func:`read_excel` would raise an error when pandas could not determine the file type, even when user specified the ``engine`` argument (:issue:`41225`) - Bug in :func:`read_clipboard` copying from an excel file shifts values into the wrong column if there are null values in first column (:issue:`41108`) -- Bug in :meth:`DataFrame.to_hdf` and :meth:`Series.to_hdf` raising a ``TypeError`` when trying to append a string column to an incompatible column +- Bug in :meth:`DataFrame.to_hdf` and :meth:`Series.to_hdf` raising a ``TypeError`` when trying to append a string column to an incompatible column (:issue:`41897`) Period ^^^^^^ diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index 934c851705e9c..352fafe015604 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -774,7 +774,7 @@ def test_append_raise(setup_path): with pytest.raises(ValueError, match=msg): store.append("df", df) - # incompatible type + # incompatible type (GH 41897) _maybe_remove(store, "df") df["foo"] = Timestamp("20130101") store.append("df", df)