Skip to content

Avoid calling S3File.s3 #27777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Aug 12, 2019
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions pandas/io/s3.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
from typing import Optional, Tuple, TYPE_CHECKING
from pandas._typing import FilePathOrBuffer

""" s3 support for remote file interactivity """
from urllib.parse import urlparse as parse_url

from pandas.compat._optional import import_optional_dependency

s3fs = import_optional_dependency(
"s3fs", extra="The s3fs package is required to handle s3 files."
)
if TYPE_CHECKING:
import s3fs
else:
s3fs = import_optional_dependency(
"s3fs", extra="The s3fs package is required to handle s3 files."
)


def _strip_schema(url):
Expand All @@ -14,7 +20,9 @@ def _strip_schema(url):
return result.netloc + result.path


def get_file_and_filesystem(filepath_or_buffer, encoding=None, mode=None):
def get_file_and_filesystem(
filepath_or_buffer: FilePathOrBuffer, mode: Optional[str] = None
) -> Tuple[s3fs.S3File, s3fs.S3FileSystem]:
from botocore.exceptions import NoCredentialsError

if mode is None:
Expand All @@ -36,9 +44,10 @@ def get_file_and_filesystem(filepath_or_buffer, encoding=None, mode=None):


def get_filepath_or_buffer(
filepath_or_buffer, encoding=None, compression=None, mode=None
):
file, _fs = get_file_and_filesystem(
filepath_or_buffer, encoding=encoding, mode=mode
)
filepath_or_buffer: FilePathOrBuffer,
encoding: Optional[str] = None,
compression: Optional[str] = None,
mode: Optional[str] = None,
) -> Tuple[s3fs.S3File, Optional[str], Optional[str], bool]:
file, _fs = get_file_and_filesystem(filepath_or_buffer, mode=mode)
return file, None, compression, True