From 2f6c901e8ade105590663f42b0723a60b84a8b39 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 21 Jan 2023 08:32:14 -0800 Subject: [PATCH 1/3] TST: Remove fsspec internals from tests --- pandas/tests/io/test_fsspec.py | 2 -- pandas/tests/io/test_gcs.py | 19 ++----------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/pandas/tests/io/test_fsspec.py b/pandas/tests/io/test_fsspec.py index 14a7d77773f29..a5790bb456d44 100644 --- a/pandas/tests/io/test_fsspec.py +++ b/pandas/tests/io/test_fsspec.py @@ -50,10 +50,8 @@ def test_read_csv(cleared_fs, df1): def test_reasonable_error(monkeypatch, cleared_fs): - from fsspec import registry from fsspec.registry import known_implementations - registry.target.clear() with pytest.raises(ValueError, match="nosuchprotocol"): read_csv("nosuchprotocol://test/test.csv") err_msg = "test error message" diff --git a/pandas/tests/io/test_gcs.py b/pandas/tests/io/test_gcs.py index e3333025da547..bb6feb0eee23e 100644 --- a/pandas/tests/io/test_gcs.py +++ b/pandas/tests/io/test_gcs.py @@ -22,12 +22,7 @@ @pytest.fixture def gcs_buffer(monkeypatch): """Emulate GCS using a binary buffer.""" - from fsspec import ( - AbstractFileSystem, - registry, - ) - - registry.target.clear() # remove state + from fsspec import AbstractFileSystem gcs_buffer = BytesIO() gcs_buffer.close = lambda: True @@ -55,9 +50,6 @@ def test_to_read_gcs(gcs_buffer, format): GH 33987 """ - from fsspec import registry - - registry.target.clear() # remove state df1 = DataFrame( { @@ -132,9 +124,6 @@ def test_to_csv_compression_encoding_gcs(gcs_buffer, compression_only, encoding) GH 35677 (to_csv, compression), GH 26124 (to_csv, encoding), and GH 32392 (read_csv, encoding) """ - from fsspec import registry - - registry.target.clear() # remove state df = tm.makeDataFrame() # reference of compressed and encoded file @@ -174,12 +163,8 @@ def test_to_csv_compression_encoding_gcs(gcs_buffer, compression_only, encoding) @td.skip_if_no("gcsfs") def test_to_parquet_gcs_new_file(monkeypatch, tmpdir): """Regression test for writing to a not-yet-existent GCS Parquet file.""" - from fsspec import ( - AbstractFileSystem, - registry, - ) + from fsspec import AbstractFileSystem - registry.target.clear() # remove state df1 = DataFrame( { "int": [1, 3], From e788164434dc2ccdbcab9e99f9fae3550f99f570 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 21 Jan 2023 10:22:27 -0800 Subject: [PATCH 2/3] update --- pandas/tests/io/test_gcs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_gcs.py b/pandas/tests/io/test_gcs.py index bb6feb0eee23e..382a70a96058f 100644 --- a/pandas/tests/io/test_gcs.py +++ b/pandas/tests/io/test_gcs.py @@ -3,6 +3,7 @@ import tarfile import zipfile +import fsspec import numpy as np import pytest @@ -37,7 +38,8 @@ def ls(self, path, **kwargs): # needed for pyarrow return [{"name": path, "type": "file"}] - monkeypatch.setattr("gcsfs.GCSFileSystem", MockGCSFileSystem) + # Overwrites the default implementation from gcsfs to our mock class + fsspec.register_implementation("gs", MockGCSFileSystem, clobber=True) return gcs_buffer From 191c2fcb7fdc2a8d5742cc81ddfc4fdd9111df8d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 21 Jan 2023 11:38:27 -0800 Subject: [PATCH 3/3] Update test_gcs.py --- pandas/tests/io/test_gcs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_gcs.py b/pandas/tests/io/test_gcs.py index 382a70a96058f..a609c1b5fc03d 100644 --- a/pandas/tests/io/test_gcs.py +++ b/pandas/tests/io/test_gcs.py @@ -3,7 +3,6 @@ import tarfile import zipfile -import fsspec import numpy as np import pytest @@ -23,12 +22,12 @@ @pytest.fixture def gcs_buffer(monkeypatch): """Emulate GCS using a binary buffer.""" - from fsspec import AbstractFileSystem + import fsspec gcs_buffer = BytesIO() gcs_buffer.close = lambda: True - class MockGCSFileSystem(AbstractFileSystem): + class MockGCSFileSystem(fsspec.AbstractFileSystem): @staticmethod def open(*args, **kwargs): gcs_buffer.seek(0)