From 1e8e7396bdee2a2537fad8516a55ca0a1f3f5a52 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 12 May 2021 13:41:36 -0700 Subject: [PATCH 1/2] Revert "Pin fastparquet to leq 0.5.0 (#41370)" This reverts commit 5db186607bca902c7f9ffffef411bd9feb8d51ac. --- ci/deps/actions-37-db.yaml | 2 +- ci/deps/azure-windows-38.yaml | 2 +- environment.yml | 2 +- requirements-dev.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/deps/actions-37-db.yaml b/ci/deps/actions-37-db.yaml index edca7b51a3420..8755e1a02c3cf 100644 --- a/ci/deps/actions-37-db.yaml +++ b/ci/deps/actions-37-db.yaml @@ -15,7 +15,7 @@ dependencies: - beautifulsoup4 - botocore>=1.11 - dask - - fastparquet>=0.4.0, <=0.5.0 + - fastparquet>=0.4.0 - fsspec>=0.7.4 - gcsfs>=0.6.0 - geopandas diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index fdea34d573340..661d8813d32d2 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -15,7 +15,7 @@ dependencies: # pandas dependencies - blosc - bottleneck - - fastparquet>=0.4.0, <=0.5.0 + - fastparquet>=0.4.0 - flask - fsspec>=0.8.0 - matplotlib=3.1.3 diff --git a/environment.yml b/environment.yml index 30fa7c0dea696..2e0228a15272e 100644 --- a/environment.yml +++ b/environment.yml @@ -99,7 +99,7 @@ dependencies: - xlwt - odfpy - - fastparquet>=0.3.2, <=0.5.0 # pandas.read_parquet, DataFrame.to_parquet + - fastparquet>=0.3.2 # pandas.read_parquet, DataFrame.to_parquet - pyarrow>=0.15.0 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - python-snappy # required by pyarrow diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e421c7715566..ea7ca43742934 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -64,7 +64,7 @@ xlrd xlsxwriter xlwt odfpy -fastparquet>=0.3.2, <=0.5.0 +fastparquet>=0.3.2 pyarrow>=0.15.0 python-snappy pyqt5>=5.9.2 From 312abfb26104eaa23772e58571d9437312c912da Mon Sep 17 00:00:00 2001 From: Thomas Li Date: Thu, 13 May 2021 21:16:54 -0700 Subject: [PATCH 2/2] Maybe fix fsspec/fastparquet? Not sure if correct --- pandas/io/parquet.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pandas/io/parquet.py b/pandas/io/parquet.py index 5ad014a334c27..34d5edee06791 100644 --- a/pandas/io/parquet.py +++ b/pandas/io/parquet.py @@ -327,9 +327,14 @@ def read( if is_fsspec_url(path): fsspec = import_optional_dependency("fsspec") - parquet_kwargs["open_with"] = lambda path, _: fsspec.open( - path, "rb", **(storage_options or {}) - ).open() + if Version(self.api.__version__) > Version("0.6.1"): + parquet_kwargs["fs"] = fsspec.open( + path, "rb", **(storage_options or {}) + ).fs + else: + parquet_kwargs["open_with"] = lambda path, _: fsspec.open( + path, "rb", **(storage_options or {}) + ).open() elif isinstance(path, str) and not os.path.isdir(path): # use get_handle only when we are very certain that it is not a directory # fsspec resources can also point to directories