3131
3232from pandas ._typing import FilePathOrBuffer
3333from pandas .compat import _get_lzma_file , _import_lzma
34+ from pandas .compat ._optional import import_optional_dependency
3435
3536from pandas .core .dtypes .common import is_file_like
3637
@@ -126,20 +127,6 @@ def stringify_path(
126127 return _expand_user (filepath_or_buffer )
127128
128129
129- def is_s3_url (url ) -> bool :
130- """Check for an s3, s3n, or s3a url"""
131- if not isinstance (url , str ):
132- return False
133- return parse_url (url ).scheme in ["s3" , "s3n" , "s3a" ]
134-
135-
136- def is_gcs_url (url ) -> bool :
137- """Check for a gcs url"""
138- if not isinstance (url , str ):
139- return False
140- return parse_url (url ).scheme in ["gcs" , "gs" ]
141-
142-
143130def urlopen (* args , ** kwargs ):
144131 """
145132 Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of
@@ -150,38 +137,24 @@ def urlopen(*args, **kwargs):
150137 return urllib .request .urlopen (* args , ** kwargs )
151138
152139
153- def get_fs_for_path ( filepath : str ) :
140+ def is_fsspec_url ( url : FilePathOrBuffer ) -> bool :
154141 """
155- Get appropriate filesystem given a filepath.
156- Supports s3fs, gcs and local file system.
157-
158- Parameters
159- ----------
160- filepath : str
161- File path. e.g s3://bucket/object, /local/path, gcs://pandas/obj
162-
163- Returns
164- -------
165- s3fs.S3FileSystem, gcsfs.GCSFileSystem, None
166- Appropriate FileSystem to use. None for local filesystem.
142+ Returns true if the given URL looks like
143+ something fsspec can handle
167144 """
168- if is_s3_url (filepath ):
169- from pandas .io import s3
170-
171- return s3 .get_fs ()
172- elif is_gcs_url (filepath ):
173- from pandas .io import gcs
174-
175- return gcs .get_fs ()
176- else :
177- return None
145+ return (
146+ isinstance (url , str )
147+ and "://" in url
148+ and not url .startswith (("http://" , "https://" ))
149+ )
178150
179151
180152def get_filepath_or_buffer (
181153 filepath_or_buffer : FilePathOrBuffer ,
182154 encoding : Optional [str ] = None ,
183155 compression : Optional [str ] = None ,
184156 mode : Optional [str ] = None ,
157+ storage_options : Optional [Dict [str , Any ]] = None ,
185158):
186159 """
187160 If the filepath_or_buffer is a url, translate and return the buffer.
@@ -194,6 +167,8 @@ def get_filepath_or_buffer(
194167 compression : {{'gzip', 'bz2', 'zip', 'xz', None}}, optional
195168 encoding : the encoding to use to decode bytes, default is 'utf-8'
196169 mode : str, optional
170+ storage_options: dict, optional
171+ passed on to fsspec, if using it; this is not yet accessed by the public API
197172
198173 Returns
199174 -------
@@ -204,6 +179,7 @@ def get_filepath_or_buffer(
204179 filepath_or_buffer = stringify_path (filepath_or_buffer )
205180
206181 if isinstance (filepath_or_buffer , str ) and is_url (filepath_or_buffer ):
182+ # TODO: fsspec can also handle HTTP via requests, but leaving this unchanged
207183 req = urlopen (filepath_or_buffer )
208184 content_encoding = req .headers .get ("Content-Encoding" , None )
209185 if content_encoding == "gzip" :
@@ -213,19 +189,23 @@ def get_filepath_or_buffer(
213189 req .close ()
214190 return reader , encoding , compression , True
215191
216- if is_s3_url (filepath_or_buffer ):
217- from pandas .io import s3
218-
219- return s3 .get_filepath_or_buffer (
220- filepath_or_buffer , encoding = encoding , compression = compression , mode = mode
221- )
222-
223- if is_gcs_url (filepath_or_buffer ):
224- from pandas .io import gcs
225-
226- return gcs .get_filepath_or_buffer (
227- filepath_or_buffer , encoding = encoding , compression = compression , mode = mode
228- )
192+ if is_fsspec_url (filepath_or_buffer ):
193+ assert isinstance (
194+ filepath_or_buffer , str
195+ ) # just to appease mypy for this branch
196+ # two special-case s3-like protocols; these have special meaning in Hadoop,
197+ # but are equivalent to just "s3" from fsspec's point of view
198+ # cc #11071
199+ if filepath_or_buffer .startswith ("s3a://" ):
200+ filepath_or_buffer = filepath_or_buffer .replace ("s3a://" , "s3://" )
201+ if filepath_or_buffer .startswith ("s3n://" ):
202+ filepath_or_buffer = filepath_or_buffer .replace ("s3n://" , "s3://" )
203+ fsspec = import_optional_dependency ("fsspec" )
204+
205+ file_obj = fsspec .open (
206+ filepath_or_buffer , mode = mode or "rb" , ** (storage_options or {})
207+ ).open ()
208+ return file_obj , encoding , compression , True
229209
230210 if isinstance (filepath_or_buffer , (str , bytes , mmap .mmap )):
231211 return _expand_user (filepath_or_buffer ), None , compression , False
0 commit comments