Skip to content

Commit 831320f

Browse files
authored
REGR: pd.to_hdf(..., dropna=True) not dropping missing rows (#37564)
1 parent 1e69e2c commit 831320f

File tree

3 files changed

+24
-5
lines changed

3 files changed

+24
-5
lines changed

doc/source/whatsnew/v1.2.0.rst

+1
Original file line numberDiff line numberDiff line change
@@ -493,6 +493,7 @@ I/O
493493
- Bug in output rendering of complex numbers showing too many trailing zeros (:issue:`36799`)
494494
- Bug in :class:`HDFStore` threw a ``TypeError`` when exporting an empty :class:`DataFrame` with ``datetime64[ns, tz]`` dtypes with a fixed HDF5 store (:issue:`20594`)
495495
- Bug in :class:`HDFStore` was dropping timezone information when exporting :class:`Series` with ``datetime64[ns, tz]`` dtypes with a fixed HDF5 store (:issue:`20594`)
496+
- Bug in :meth:`DataFrame.to_hdf` was not dropping missing rows with ``dropna=True`` (:issue:`35719`)
496497

497498
Plotting
498499
^^^^^^^^

pandas/io/pytables.py

+3
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,7 @@ def to_hdf(
268268
data_columns=data_columns,
269269
errors=errors,
270270
encoding=encoding,
271+
dropna=dropna,
271272
)
272273

273274
path_or_buf = stringify_path(path_or_buf)
@@ -1051,6 +1052,7 @@ def put(
10511052
encoding=None,
10521053
errors: str = "strict",
10531054
track_times: bool = True,
1055+
dropna: bool = False,
10541056
):
10551057
"""
10561058
Store object in HDFStore.
@@ -1100,6 +1102,7 @@ def put(
11001102
encoding=encoding,
11011103
errors=errors,
11021104
track_times=track_times,
1105+
dropna=dropna,
11031106
)
11041107

11051108
def remove(self, key: str, where=None, start=None, stop=None):

pandas/tests/io/pytables/test_store.py

+20-5
Original file line numberDiff line numberDiff line change
@@ -1253,17 +1253,32 @@ def test_append_all_nans(self, setup_path):
12531253
store.append("df2", df[10:], dropna=False)
12541254
tm.assert_frame_equal(store["df2"], df)
12551255

1256-
# Test to make sure defaults are to not drop.
1257-
# Corresponding to Issue 9382
1256+
def test_store_dropna(self, setup_path):
12581257
df_with_missing = DataFrame(
1259-
{"col1": [0, np.nan, 2], "col2": [1, np.nan, np.nan]}
1258+
{"col1": [0.0, np.nan, 2.0], "col2": [1.0, np.nan, np.nan]},
1259+
index=list("abc"),
12601260
)
1261+
df_without_missing = DataFrame(
1262+
{"col1": [0.0, 2.0], "col2": [1.0, np.nan]}, index=list("ac")
1263+
)
1264+
1265+
# # Test to make sure defaults are to not drop.
1266+
# # Corresponding to Issue 9382
1267+
with ensure_clean_path(setup_path) as path:
1268+
df_with_missing.to_hdf(path, "df", format="table")
1269+
reloaded = read_hdf(path, "df")
1270+
tm.assert_frame_equal(df_with_missing, reloaded)
12611271

12621272
with ensure_clean_path(setup_path) as path:
1263-
df_with_missing.to_hdf(path, "df_with_missing", format="table")
1264-
reloaded = read_hdf(path, "df_with_missing")
1273+
df_with_missing.to_hdf(path, "df", format="table", dropna=False)
1274+
reloaded = read_hdf(path, "df")
12651275
tm.assert_frame_equal(df_with_missing, reloaded)
12661276

1277+
with ensure_clean_path(setup_path) as path:
1278+
df_with_missing.to_hdf(path, "df", format="table", dropna=True)
1279+
reloaded = read_hdf(path, "df")
1280+
tm.assert_frame_equal(df_without_missing, reloaded)
1281+
12671282
def test_read_missing_key_close_store(self, setup_path):
12681283
# GH 25766
12691284
with ensure_clean_path(setup_path) as path:

0 commit comments

Comments
 (0)