Merge branch 'master' of https://github.com/pandas-dev/pandas into Replace_wildcardimports_in_toplevelinit_as_precursor_for_reshape,stata,io_PRs_#25936_#25940_#25939

Author: ryankarlos

doc/source/whatsnew/v0.25.0.rst

@@ -361,6 +361,7 @@ I/O
 - Bug in ``read_csv`` which would not raise ``ValueError`` if a column index in ``usecols`` was out of bounds (:issue:`25623`)
 - Improved the explanation for the failure when value labels are repeated in Stata dta files and suggested work-arounds (:issue:`25772`)
 - Improved :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` to read incorrectly formatted 118 format files saved by Stata (:issue:`25960`)
+- Fixed bug in loading objects from S3 that contain ``#`` characters in the URL (:issue:`25945`)
 
 Plotting
 ^^^^^^^^

mypy.ini

@@ -167,18 +167,6 @@ ignore_errors=True
 [mypy-pandas.io.stata]
 ignore_errors=True
 
-[mypy-pandas.plotting._core]
-ignore_errors=True
-
-[mypy-pandas.tseries.frequencies]
-ignore_errors=True
-
-[mypy-pandas.tseries.holiday]
-ignore_errors=True
-
-[mypy-pandas.tseries.offsets]
-ignore_errors=True
-
 [mypy-pandas.util._doctools]
 ignore_errors=True
 

pandas/compat/__init__.py

@@ -8,7 +8,6 @@
 * lists: lrange(), lmap(), lzip(), lfilter()
 * iterable method compatibility: iteritems, iterkeys, itervalues
   * Uses the original method if available, otherwise uses items, keys, values.
-* bind_method: binds functions to classes
 * add_metaclass(metaclass) - class decorator that recreates class with with the
   given metaclass instead (and avoids intermediary class creation)
 
@@ -22,10 +21,7 @@
 from distutils.version import LooseVersion
 import sys
 import platform
-import types
 import struct
-import inspect
-from collections import namedtuple
 
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] >= 3
@@ -34,8 +30,6 @@
 PY37 = sys.version_info >= (3, 7)
 PYPY = platform.python_implementation() == 'PyPy'
 
-from pandas.compat.chainmap import DeepChainMap
-
 
 # list-producing versions of the major Python iterating functions
 def lrange(*args, **kwargs):
@@ -54,58 +48,6 @@ def lfilter(*args, **kwargs):
     return list(filter(*args, **kwargs))
 
 
-if PY3:
-    def isidentifier(s):
-        return s.isidentifier()
-
-    def str_to_bytes(s, encoding=None):
-        return s.encode(encoding or 'ascii')
-
-    def bytes_to_str(b, encoding=None):
-        return b.decode(encoding or 'utf-8')
-
-    # The signature version below is directly copied from Django,
-    # https://github.com/django/django/pull/4846
-    def signature(f):
-        sig = inspect.signature(f)
-        args = [
-            p.name for p in sig.parameters.values()
-            if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
-        ]
-        varargs = [
-            p.name for p in sig.parameters.values()
-            if p.kind == inspect.Parameter.VAR_POSITIONAL
-        ]
-        varargs = varargs[0] if varargs else None
-        keywords = [
-            p.name for p in sig.parameters.values()
-            if p.kind == inspect.Parameter.VAR_KEYWORD
-        ]
-        keywords = keywords[0] if keywords else None
-        defaults = [
-            p.default for p in sig.parameters.values()
-            if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
-            and p.default is not p.empty
-        ] or None
-        argspec = namedtuple('Signature', ['args', 'defaults',
-                                           'varargs', 'keywords'])
-        return argspec(args, defaults, varargs, keywords)
-else:
-    # Python 2
-    _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
-
-    def isidentifier(s, dotted=False):
-        return bool(_name_re.match(s))
-
-    def str_to_bytes(s, encoding='ascii'):
-        return s
-
-    def bytes_to_str(b, encoding='ascii'):
-        return b
-
-    def signature(f):
-        return inspect.getargspec(f)
-
 if PY2:
     def iteritems(obj, **kw):
         return obj.iteritems(**kw)
@@ -126,30 +68,6 @@ def iterkeys(obj, **kw):
     def itervalues(obj, **kw):
         return iter(obj.values(**kw))
 
-
-def bind_method(cls, name, func):
-    """Bind a method to class, python 2 and python 3 compatible.
-
-    Parameters
-    ----------
-
-    cls : type
-        class to receive bound method
-    name : basestring
-        name of method on class instance
-    func : function
-        function to be bound as method
-
-
-    Returns
-    -------
-    None
-    """
-    # only python 2 has bound/unbound method issue
-    if not PY3:
-        setattr(cls, name, types.MethodType(func, None, cls))
-    else:
-        setattr(cls, name, func)
 # ----------------------------------------------------------------------------
 # functions largely based / taken from the six module
 
@@ -164,7 +82,7 @@ def to_str(s):
         Convert bytes and non-string into Python 3 str
         """
         if isinstance(s, bytes):
-            s = bytes_to_str(s)
+            s = s.decode('utf-8')
         elif not isinstance(s, str):
             s = str(s)
         return s
@@ -203,6 +121,7 @@ def wrapper(cls):
         return metaclass(cls.__name__, cls.__bases__, orig_vars)
     return wrapper
 
+
 if PY3:
     def raise_with_traceback(exc, traceback=Ellipsis):
         if traceback == Ellipsis:
@@ -238,6 +157,7 @@ def raise_with_traceback(exc, traceback=Ellipsis):
 else:
     re_type = type(re.compile(''))
 
+
 # https://github.com/pandas-dev/pandas/pull/9123
 def is_platform_little_endian():
     """ am I little endian """

pandas/core/apply.py

@@ -1,9 +1,9 @@
+import inspect
 import warnings
 
 import numpy as np
 
 from pandas._libs import reduction
-import pandas.compat as compat
 from pandas.util._decorators import cache_readonly
 
 from pandas.core.dtypes.common import (
@@ -123,7 +123,7 @@ def get_result(self):
             # Some methods (shift, etc.) require the axis argument, others
             # don't, so inspect and insert if necessary.
             func = getattr(self.obj, self.f)
-            sig = compat.signature(func)
+            sig = inspect.getfullargspec(func)
             if 'axis' in sig.args:
                 self.kwds['axis'] = self.axis
             return func(*self.args, **self.kwds)

pandas/core/computation/pytables.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from pandas._libs.tslibs import Timedelta, Timestamp
-from pandas.compat import DeepChainMap
+from pandas.compat.chainmap import DeepChainMap
 
 from pandas.core.dtypes.common import is_list_like
 

pandas/core/computation/scope.py

@@ -13,7 +13,7 @@
 import numpy as np
 
 from pandas._libs.tslibs import Timestamp
-from pandas.compat import DeepChainMap
+from pandas.compat.chainmap import DeepChainMap
 
 from pandas.core.base import StringMixin
 import pandas.core.computation as compu

pandas/core/generic.py

@@ -16,7 +16,7 @@
 
 from pandas._libs import Timestamp, iNaT, properties
 import pandas.compat as compat
-from pandas.compat import isidentifier, lrange, lzip, set_function_name, to_str
+from pandas.compat import lrange, lzip, set_function_name, to_str
 from pandas.compat.numpy import function as nv
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import (
@@ -5150,7 +5150,7 @@ def _dir_additions(self):
         If info_axis is a MultiIndex, it's first level values are used.
         """
         additions = {c for c in self._info_axis.unique(level=0)[:100]
-                     if isinstance(c, str) and isidentifier(c)}
+                     if isinstance(c, str) and c.isidentifier()}
         return super(NDFrame, self)._dir_additions().union(additions)
 
     # ----------------------------------------------------------------------
@@ -8774,22 +8774,22 @@ def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
             .. versionadded:: 0.18.1
                 A callable can be used as other.
 
-        inplace : boolean, default False
+        inplace : bool, default False
             Whether to perform the operation in place on the data.
         axis : int, default None
             Alignment axis if needed.
         level : int, default None
             Alignment level if needed.
-        errors : str, {'raise', 'ignore'}, default `raise`
+        errors : str, {'raise', 'ignore'}, default 'raise'
             Note that currently this parameter won't affect
             the results and will always coerce to a suitable dtype.
 
-            - `raise` : allow exceptions to be raised.
-            - `ignore` : suppress exceptions. On error return original object.
+            - 'raise' : allow exceptions to be raised.
+            - 'ignore' : suppress exceptions. On error return original object.
 
-        try_cast : boolean, default False
+        try_cast : bool, default False
             Try to cast the result back to the input type (if possible).
-        raise_on_error : boolean, default True
+        raise_on_error : bool, default True
             Whether to raise on invalid data types (e.g. trying to where on
             strings).
 
@@ -8799,7 +8799,7 @@ def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
 
         Returns
         -------
-        wh : same type as caller
+        Same type as caller
 
         See Also
         --------
@@ -8848,6 +8848,13 @@ def _where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
         dtype: int64
 
         >>> df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=['A', 'B'])
+        >>> df
+           A  B
+        0  0  1
+        1  2  3
+        2  4  5
+        3  6  7
+        4  8  9
         >>> m = df %% 3 == 0
         >>> df.where(m, -df)
            A  B

pandas/core/ops.py

@@ -12,7 +12,6 @@
 
 from pandas._libs import algos as libalgos, lib, ops as libops
 import pandas.compat as compat
-from pandas.compat import bind_method
 from pandas.errors import NullFrequencyError
 from pandas.util._decorators import Appender
 
@@ -1545,7 +1544,7 @@ def add_methods(cls, new_methods):
         force = not (issubclass(cls, ABCSparseArray) and
                      name.startswith('__i'))
         if force or name not in cls.__dict__:
-            bind_method(cls, name, method)
+            setattr(cls, name, method)
 
 
 # ----------------------------------------------------------------------

pandas/io/clipboards.py

@@ -2,8 +2,6 @@
 from io import StringIO
 import warnings
 
-import pandas.compat as compat
-
 from pandas.core.dtypes.generic import ABCDataFrame
 
 from pandas import get_option, option_context
@@ -38,10 +36,8 @@ def read_clipboard(sep=r'\s+', **kwargs):  # pragma: no cover
 
     # Try to decode (if needed, as "text" might already be a string here).
     try:
-        text = compat.bytes_to_str(
-            text, encoding=(kwargs.get('encoding') or
-                            get_option('display.encoding'))
-        )
+        text = text.decode(kwargs.get('encoding')
+                           or get_option('display.encoding'))
     except AttributeError:
         pass
 

pandas/io/common.py

@@ -16,7 +16,6 @@
 from urllib.request import pathname2url, urlopen
 import zipfile
 
-import pandas.compat as compat
 from pandas.errors import (  # noqa
     AbstractMethodError, DtypeWarning, EmptyDataError, ParserError,
     ParserWarning)
@@ -460,7 +459,7 @@ def __next__(self):
 
         # readline returns bytes, not str, but Python's CSV reader
         # expects str, so convert the output to str before continuing
-        newline = compat.bytes_to_str(newline)
+        newline = newline.decode('utf-8')
 
         # mmap doesn't raise if reading past the allocated
         # data but instead returns an empty string, so raise

pandas/io/s3.py

@@ -10,7 +10,7 @@
 
 def _strip_schema(url):
     """Returns the url without the s3:// part"""
-    result = parse_url(url)
+    result = parse_url(url, allow_fragments=False)
     return result.netloc + result.path
 
 

pandas/plotting/_core.py

@@ -2,6 +2,7 @@
 # pylint: disable=E1101
 from collections import namedtuple
 import re
+from typing import List, Optional, Type
 import warnings
 
 import numpy as np
@@ -78,7 +79,7 @@ def _kind(self):
 
     _layout_type = 'vertical'
     _default_rot = 0
-    orientation = None
+    orientation = None  # type: Optional[str]
     _pop_attributes = ['label', 'style', 'logy', 'logx', 'loglog',
                        'mark_right', 'stacked']
     _attr_defaults = {'logy': False, 'logx': False, 'loglog': False,
@@ -1723,7 +1724,8 @@ def result(self):
 _all_kinds = _common_kinds + _dataframe_kinds + _series_kinds
 
 _klasses = [LinePlot, BarPlot, BarhPlot, KdePlot, HistPlot, BoxPlot,
-            ScatterPlot, HexBinPlot, AreaPlot, PiePlot]
+            ScatterPlot, HexBinPlot, AreaPlot, PiePlot] \
+    # type: List[Type[MPLPlot]]
 
 _plot_klass = {klass._kind: klass for klass in _klasses}
 

pandas/tests/io/conftest.py

@@ -59,6 +59,7 @@ def s3_resource(tips_file, jsonl_file):
         moto = pytest.importorskip('moto')
 
         test_s3_files = [
+            ('tips#1.csv', tips_file),
             ('tips.csv', tips_file),
             ('tips.csv.gz', tips_file + '.gz'),
             ('tips.csv.bz2', tips_file + '.bz2'),

pandas/tests/io/formats/test_printing.py

@@ -5,7 +5,6 @@
 import pandas._config.config as cf
 
 import pandas as pd
-from pandas import compat
 
 import pandas.io.formats.format as fmt
 import pandas.io.formats.printing as printing
@@ -27,7 +26,7 @@ def test_repr_binary_type():
         raw = bytes(letters, encoding=cf.get_option('display.encoding'))
     except TypeError:
         raw = bytes(letters)
-    b = str(compat.bytes_to_str(raw))
+    b = str(raw.decode('utf-8'))
     res = printing.pprint_thing(b, quote_strings=True)
     assert res == repr(b)
     res = printing.pprint_thing(b, quote_strings=False)