Backport PR pandas-dev#41897: BUG: to_hdf append string column to incompatible column (pandas-dev#42074)

Co-authored-by: lrepiton <[email protected]>

Author: meeseeksmachine

doc/source/whatsnew/v1.3.0.rst

@@ -1080,6 +1080,7 @@ I/O
 - Bug in the conversion from PyArrow to pandas (e.g. for reading Parquet) with nullable dtypes and a PyArrow array whose data buffer size is not a multiple of the dtype size (:issue:`40896`)
 - Bug in :func:`read_excel` would raise an error when pandas could not determine the file type even though the user specified the ``engine`` argument (:issue:`41225`)
 - Bug in :func:`read_clipboard` copying from an excel file shifts values into the wrong column if there are null values in first column (:issue:`41108`)
+- Bug in :meth:`DataFrame.to_hdf` and :meth:`Series.to_hdf` raising a ``TypeError`` when trying to append a string column to an incompatible column (:issue:`41897`)
 
 Period
 ^^^^^^

pandas/io/pytables.py

@@ -5000,7 +5000,7 @@ def _maybe_convert_for_string_atom(
     # check for column in the values conflicts
     if existing_col is not None:
         eci = existing_col.validate_col(itemsize)
-        if eci > itemsize:
+        if eci is not None and eci > itemsize:
             itemsize = eci
 
     data_converted = data_converted.astype(f"|S{itemsize}", copy=False)

pandas/tests/io/pytables/test_append.py

@@ -778,6 +778,22 @@ def test_append_raise(setup_path):
         with pytest.raises(ValueError, match=msg):
             store.append("df", df)
 
+        # incompatible type (GH 41897)
+        _maybe_remove(store, "df")
+        df["foo"] = Timestamp("20130101")
+        store.append("df", df)
+        df["foo"] = "bar"
+        msg = re.escape(
+            "invalid combination of [values_axes] on appending data "
+            "[name->values_block_1,cname->values_block_1,"
+            "dtype->bytes24,kind->string,shape->(1, 30)] "
+            "vs current table "
+            "[name->values_block_1,cname->values_block_1,"
+            "dtype->datetime64,kind->datetime64,shape->None]"
+        )
+        with pytest.raises(ValueError, match=msg):
+            store.append("df", df)
+
 
 def test_append_with_timedelta(setup_path):
     # GH 3577