From c280b2c7bf085d0e8b0e2ab03ee31164d42a85c1 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 15 Jun 2020 17:24:59 +0200 Subject: [PATCH 1/2] TST: ensure read_parquet filter argument is correctly passed though (pyarrow engine) --- pandas/tests/io/test_parquet.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index 7ee551194bf76..380f3f573c940 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -671,6 +671,18 @@ def test_timestamp_nanoseconds(self, pa): df = pd.DataFrame({"a": pd.date_range("2017-01-01", freq="1n", periods=10)}) check_round_trip(df, pa, write_kwargs={"version": "2.0"}) + @td.skip_if_no("pyarrow", min_version="0.17") + def test_filter_row_groups(self, pa): + # https://github.com/pandas-dev/pandas/issues/26551 + d = {"a": list(range(0, 3))} + df = pd.DataFrame(d) + with tm.ensure_clean() as path: + df.to_parquet(path, pa) + result = read_parquet( + path, pa, filters=[("a", "==", 0)], use_legacy_dataset=False + ) + assert len(result) == 1 + class TestParquetFastParquet(Base): @td.skip_if_no("fastparquet", min_version="0.3.2") From dd0323ba1cb590faf2f3cbc456d62deecd053dbb Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 15 Jun 2020 17:27:44 +0200 Subject: [PATCH 2/2] clean-up --- pandas/tests/io/test_parquet.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index 380f3f573c940..efd34c58d7d19 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -674,8 +674,7 @@ def test_timestamp_nanoseconds(self, pa): @td.skip_if_no("pyarrow", min_version="0.17") def test_filter_row_groups(self, pa): # https://github.com/pandas-dev/pandas/issues/26551 - d = {"a": list(range(0, 3))} - df = pd.DataFrame(d) + df = pd.DataFrame({"a": list(range(0, 3))}) with tm.ensure_clean() as path: df.to_parquet(path, pa) result = read_parquet(