diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 8905560aa2108..99b408bac8a9f 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -111,8 +111,10 @@ fi if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then # Check for imports from pandas.core.common instead of `import pandas.core.common as com` + # Check for imports from collections.abc instead of `from collections import abc` MSG='Check for non-standard imports' ; echo $MSG invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas + invgrep -R --include="*.py*" -E "from collections.abc import " pandas # invgrep -R --include="*.py*" -E "from numpy import nan " pandas # GH#24822 not yet implemented since the offending imports have not all been removed RET=$(($RET + $?)) ; echo $MSG "DONE" diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index caf70a32e8d19..428a4f957f231 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -29,7 +29,6 @@ import struct import inspect from collections import namedtuple -import collections PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] >= 3 @@ -106,16 +105,6 @@ def lmap(*args, **kwargs): def lfilter(*args, **kwargs): return list(filter(*args, **kwargs)) - - Hashable = collections.abc.Hashable - Iterable = collections.abc.Iterable - Iterator = collections.abc.Iterator - Mapping = collections.abc.Mapping - MutableMapping = collections.abc.MutableMapping - Sequence = collections.abc.Sequence - Sized = collections.abc.Sized - Set = collections.abc.Set - else: # Python 2 _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$") @@ -138,15 +127,6 @@ def signature(f): lmap = builtins.map lfilter = builtins.filter - Hashable = collections.Hashable - Iterable = collections.Iterable - Iterator = collections.Iterator - Mapping = collections.Mapping - MutableMapping = collections.MutableMapping - Sequence = collections.Sequence - Sized = collections.Sized - Set = collections.Set - if PY2: def iteritems(obj, **kw): return obj.iteritems(**kw) diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index 3d18255e78aa9..288035d824b8b 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -1,6 +1,7 @@ """ SparseArray data structure """ +from collections import abc import numbers import operator import re @@ -1377,7 +1378,7 @@ def map(self, mapper): if isinstance(mapper, ABCSeries): mapper = mapper.to_dict() - if isinstance(mapper, compat.Mapping): + if isinstance(mapper, abc.Mapping): fill_value = mapper.get(self.fill_value, self.fill_value) sp_values = [mapper.get(x, None) for x in self.sp_values] else: diff --git a/pandas/core/common.py b/pandas/core/common.py index c3cef3ff44896..3cb23e9ee921d 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -5,7 +5,7 @@ """ import collections -from collections import OrderedDict +from collections import OrderedDict, abc from datetime import datetime, timedelta from functools import partial import inspect @@ -14,7 +14,6 @@ import numpy as np from pandas._libs import lib, tslibs -import pandas.compat as compat from pandas.compat import PY36, iteritems from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike @@ -374,13 +373,13 @@ def standardize_mapping(into): Parameters ---------- - into : instance or subclass of collections.Mapping + into : instance or subclass of collections.abc.Mapping Must be a class, an initialized collections.defaultdict, - or an instance of a collections.Mapping subclass. + or an instance of a collections.abc.Mapping subclass. Returns ------- - mapping : a collections.Mapping subclass or other constructor + mapping : a collections.abc.Mapping subclass or other constructor a callable object that can accept an iterator to create the desired Mapping. @@ -394,7 +393,7 @@ def standardize_mapping(into): return partial( collections.defaultdict, into.default_factory) into = type(into) - if not issubclass(into, compat.Mapping): + if not issubclass(into, abc.Mapping): raise TypeError('unsupported type: {into}'.format(into=into)) elif into == collections.defaultdict: raise TypeError( @@ -471,7 +470,7 @@ def _get_rename_function(mapper): Returns a function that will map names/labels, dependent if mapper is a dict, Series or just a function. """ - if isinstance(mapper, (compat.Mapping, ABCSeries)): + if isinstance(mapper, (abc.Mapping, ABCSeries)): def f(x): if x in mapper: diff --git a/pandas/core/dtypes/inference.py b/pandas/core/dtypes/inference.py index afd2501ac2da1..09d585ee3899b 100644 --- a/pandas/core/dtypes/inference.py +++ b/pandas/core/dtypes/inference.py @@ -1,14 +1,13 @@ """ basic inference routines """ +from collections import abc from numbers import Number import re import numpy as np from pandas._libs import lib -from pandas.compat import PY2, Set, re_type - -from pandas import compat +from pandas.compat import PY2, re_type is_bool = lib.is_bool @@ -113,7 +112,7 @@ def _iterable_not_string(obj): False """ - return isinstance(obj, compat.Iterable) and not isinstance(obj, str) + return isinstance(obj, abc.Iterable) and not isinstance(obj, str) def is_iterator(obj): @@ -288,7 +287,7 @@ def is_list_like(obj, allow_sets=True): False """ - return (isinstance(obj, compat.Iterable) and + return (isinstance(obj, abc.Iterable) and # we do not count strings/unicode/bytes as list-like not isinstance(obj, (str, bytes)) and @@ -296,7 +295,7 @@ def is_list_like(obj, allow_sets=True): not (isinstance(obj, np.ndarray) and obj.ndim == 0) and # exclude sets if allow_sets is False - not (allow_sets is False and isinstance(obj, Set))) + not (allow_sets is False and isinstance(obj, abc.Set))) def is_array_like(obj): @@ -436,8 +435,8 @@ def is_named_tuple(obj): def is_hashable(obj): """Return True if hash(obj) will succeed, False otherwise. - Some types will pass a test against collections.Hashable but fail when they - are actually hashed with hash(). + Some types will pass a test against collections.abc.Hashable but fail when + they are actually hashed with hash(). Distinguish between these and other types by trying the call to hash() and seeing if they raise TypeError. @@ -445,14 +444,14 @@ def is_hashable(obj): Examples -------- >>> a = ([],) - >>> isinstance(a, collections.Hashable) + >>> isinstance(a, collections.abc.Hashable) True >>> is_hashable(a) False """ - # Unfortunately, we can't use isinstance(obj, collections.Hashable), which - # can be faster than calling hash. That is because numpy scalars on Python - # 3 fail this test. + # Unfortunately, we can't use isinstance(obj, collections.abc.Hashable), + # which can be faster than calling hash. That is because numpy scalars + # fail this test. # Reconsider this decision once this numpy bug is fixed: # https://github.com/numpy/numpy/issues/5562 diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 6ca42130e86e5..88637e4fb456d 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -11,7 +11,7 @@ labeling information """ import collections -from collections import OrderedDict +from collections import OrderedDict, abc import functools import itertools import sys @@ -33,8 +33,7 @@ validate_axis_style_args) from pandas import compat -from pandas.compat import ( - PY36, Iterator, StringIO, lmap, lzip, raise_with_traceback) +from pandas.compat import PY36, StringIO, lmap, lzip, raise_with_traceback from pandas.compat.numpy import function as nv from pandas.core.dtypes.cast import ( maybe_upcast, @@ -426,9 +425,9 @@ def __init__(self, data=None, index=None, columns=None, dtype=None, copy=copy) # For data is list-like, or Iterable (will consume into list) - elif (isinstance(data, compat.Iterable) and + elif (isinstance(data, abc.Iterable) and not isinstance(data, (str, bytes))): - if not isinstance(data, compat.Sequence): + if not isinstance(data, abc.Sequence): data = list(data) if len(data) > 0: if is_list_like(data[0]) and getattr(data[0], 'ndim', 1) == 1: @@ -1203,7 +1202,7 @@ def to_dict(self, orient='dict', into=dict): indicates `split`. into : class, default dict - The collections.Mapping subclass used for all Mappings + The collections.abc.Mapping subclass used for all Mappings in the return value. Can be the actual class or an empty instance of the mapping type you want. If you want a collections.defaultdict, you must pass it initialized. @@ -1212,8 +1211,8 @@ def to_dict(self, orient='dict', into=dict): Returns ------- - dict, list or collections.Mapping - Return a collections.Mapping object representing the DataFrame. + dict, list or collections.abc.Mapping + Return a collections.abc.Mapping object representing the DataFrame. The resulting transformation depends on the `orient` parameter. See Also @@ -4080,7 +4079,7 @@ def set_index(self, keys, drop=True, append=False, inplace=False, the same length as the calling DataFrame, or a list containing an arbitrary combination of column keys and arrays. Here, "array" encompasses :class:`Series`, :class:`Index`, ``np.ndarray``, and - instances of :class:`abc.Iterator`. + instances of :class:`~collections.abc.Iterator`. drop : bool, default True Delete columns to be used as the new index. append : bool, default False @@ -4166,7 +4165,7 @@ def set_index(self, keys, drop=True, append=False, inplace=False, missing = [] for col in keys: if isinstance(col, (ABCIndexClass, ABCSeries, np.ndarray, - list, Iterator)): + list, abc.Iterator)): # arrays are fine as long as they are one-dimensional # iterators get converted to list below if getattr(col, 'ndim', 1) != 1: @@ -4213,7 +4212,7 @@ def set_index(self, keys, drop=True, append=False, inplace=False, elif isinstance(col, (list, np.ndarray)): arrays.append(col) names.append(None) - elif isinstance(col, Iterator): + elif isinstance(col, abc.Iterator): arrays.append(list(col)) names.append(None) # from here, col can only be a column label diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 4e441f26ea60c..b6fc31bb6f015 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -6,7 +6,7 @@ which here returns a DataFrameGroupBy object. """ -import collections +from collections import OrderedDict, abc import copy from functools import partial from textwrap import dedent @@ -226,7 +226,7 @@ def _aggregate_generic(self, func, *args, **kwargs): axis = self.axis obj = self._obj_with_exclusions - result = collections.OrderedDict() + result = OrderedDict() if axis != obj._info_axis_number: try: for name, data in self: @@ -253,7 +253,7 @@ def _aggregate_item_by_item(self, func, *args, **kwargs): # only for axis==0 obj = self._obj_with_exclusions - result = collections.OrderedDict() + result = OrderedDict() cannot_agg = [] errors = None for item in obj: @@ -770,7 +770,7 @@ def aggregate(self, func_or_funcs, *args, **kwargs): if isinstance(func_or_funcs, str): return getattr(self, func_or_funcs)(*args, **kwargs) - if isinstance(func_or_funcs, compat.Iterable): + if isinstance(func_or_funcs, abc.Iterable): # Catch instances of lists / tuples # but not the class list / tuple itself. ret = self._aggregate_multiple_funcs(func_or_funcs, @@ -834,7 +834,7 @@ def _aggregate_multiple_funcs(self, arg, _level): columns.append(com.get_callable_name(f)) arg = lzip(columns, arg) - results = collections.OrderedDict() + results = OrderedDict() for name, func in arg: obj = self if name in results: @@ -911,7 +911,7 @@ def _get_index(): name=self._selection_name) def _aggregate_named(self, func, *args, **kwargs): - result = collections.OrderedDict() + result = OrderedDict() for name, group in self: group.name = name @@ -1507,7 +1507,7 @@ def _apply_to_column_groupbys(self, func): def _fill(self, direction, limit=None): """Overridden method to join grouped columns in output""" res = super(DataFrameGroupBy, self)._fill(direction, limit=limit) - output = collections.OrderedDict( + output = OrderedDict( (grp.name, grp.grouper) for grp in self.grouper.groupings) from pandas import concat diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py index 684c8ff8945a3..924e914392839 100644 --- a/pandas/core/internals/construction.py +++ b/pandas/core/internals/construction.py @@ -2,14 +2,13 @@ Functions for preparing various inputs passed to the DataFrame or Series constructors before passing them to a BlockManager. """ -from collections import OrderedDict +from collections import OrderedDict, abc import numpy as np import numpy.ma as ma from pandas._libs import lib from pandas._libs.tslibs import IncompatibleFrequency -import pandas.compat as compat from pandas.compat import lmap, lrange, raise_with_traceback from pandas.core.dtypes.cast import ( @@ -395,7 +394,7 @@ def to_arrays(data, columns, coerce_float=False, dtype=None): if isinstance(data[0], (list, tuple)): return _list_to_arrays(data, columns, coerce_float=coerce_float, dtype=dtype) - elif isinstance(data[0], compat.Mapping): + elif isinstance(data[0], abc.Mapping): return _list_of_dict_to_arrays(data, columns, coerce_float=coerce_float, dtype=dtype) elif isinstance(data[0], ABCSeries): diff --git a/pandas/core/series.py b/pandas/core/series.py index 1a7d9ff70be3a..a176e4c0b8ae9 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -1,7 +1,7 @@ """ Data structure for 1-dimensional cross-sectional and time series data """ -from collections import OrderedDict +from collections import OrderedDict, abc from shutil import get_terminal_size from textwrap import dedent import warnings @@ -224,8 +224,8 @@ def __init__(self, data=None, index=None, dtype=None, name=None, raise TypeError("{0!r} type is unordered" "".format(data.__class__.__name__)) # If data is Iterable but not list-like, consume into list. - elif (isinstance(data, compat.Iterable) - and not isinstance(data, compat.Sized)): + elif (isinstance(data, abc.Iterable) and + not isinstance(data, abc.Sized)): data = list(data) else: @@ -1496,7 +1496,7 @@ def to_dict(self, into=dict): Parameters ---------- into : class, default dict - The collections.Mapping subclass to use as the return + The collections.abc.Mapping subclass to use as the return object. Can be the actual class or an empty instance of the mapping type you want. If you want a collections.defaultdict, you must pass it initialized. @@ -1505,7 +1505,7 @@ def to_dict(self, into=dict): Returns ------- - collections.Mapping + collections.abc.Mapping Key-value representation of Series. Examples diff --git a/pandas/core/sparse/series.py b/pandas/core/sparse/series.py index db4d3e876dec5..c5b07c7b6c881 100644 --- a/pandas/core/sparse/series.py +++ b/pandas/core/sparse/series.py @@ -5,6 +5,7 @@ # pylint: disable=E1101,E1103,W0231 +from collections import abc import warnings import numpy as np @@ -12,7 +13,6 @@ import pandas._libs.index as libindex import pandas._libs.sparse as splib from pandas._libs.sparse import BlockIndex, IntIndex -import pandas.compat as compat from pandas.compat.numpy import function as nv from pandas.util._decorators import Appender, Substitution @@ -82,7 +82,7 @@ def __init__(self, data=None, index=None, sparse_index=None, kind='block', if index is not None: data = data.reindex(index) - elif isinstance(data, compat.Mapping): + elif isinstance(data, abc.Mapping): data, index = Series()._init_dict(data, index=index) elif is_scalar(data) and index is not None: diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index 0b7a0965cce2e..66d563a7c6f85 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -1,3 +1,4 @@ +from collections import abc from datetime import datetime, time from functools import partial @@ -17,7 +18,6 @@ from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries from pandas.core.dtypes.missing import notna -from pandas import compat from pandas.core import algorithms @@ -599,7 +599,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False, else: values = convert_listlike(arg._values, True, format) result = arg._constructor(values, index=arg.index, name=arg.name) - elif isinstance(arg, (ABCDataFrame, compat.MutableMapping)): + elif isinstance(arg, (ABCDataFrame, abc.MutableMapping)): result = _assemble_from_unit_mappings(arg, errors, box, tz) elif isinstance(arg, ABCIndexClass): cache_array = _maybe_cache(arg, format, cache, convert_listlike) diff --git a/pandas/io/html.py b/pandas/io/html.py index 80cc1721d758f..1d588632b69f8 100644 --- a/pandas/io/html.py +++ b/pandas/io/html.py @@ -3,12 +3,12 @@ """ +from collections import abc from distutils.version import LooseVersion import numbers import os import re -import pandas.compat as compat from pandas.compat import iteritems, lmap, lrange, raise_with_traceback from pandas.errors import AbstractMethodError, EmptyDataError @@ -857,7 +857,7 @@ def _validate_flavor(flavor): flavor = 'lxml', 'bs4' elif isinstance(flavor, str): flavor = flavor, - elif isinstance(flavor, compat.Iterable): + elif isinstance(flavor, abc.Iterable): if not all(isinstance(flav, str) for flav in flavor): raise TypeError('Object of type {typ!r} is not an iterable of ' 'strings' diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 0df6631a1e9d7..0a1c79292b1d4 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -2,6 +2,7 @@ # Arithmetc tests for DataFrame/Series/Index/Array classes that should # behave identically. # Specifically for numeric dtypes +from collections import abc from decimal import Decimal from itertools import combinations import operator @@ -9,8 +10,6 @@ import numpy as np import pytest -from pandas.compat import Iterable - import pandas as pd from pandas import Index, Series, Timedelta, TimedeltaIndex from pandas.core import ops @@ -808,7 +807,7 @@ def check_comparators(series, other, check_dtype=True): def test_divmod(self): def check(series, other): results = divmod(series, other) - if isinstance(other, Iterable) and len(series) != len(other): + if isinstance(other, abc.Iterable) and len(series) != len(other): # if the lengths don't match, this is the test where we use # `tser[::2]`. Pad every other value in `other_np` with nan. other_np = [] diff --git a/pandas/tests/dtypes/test_inference.py b/pandas/tests/dtypes/test_inference.py index 9731b9b63d028..cca40244f2f3d 100644 --- a/pandas/tests/dtypes/test_inference.py +++ b/pandas/tests/dtypes/test_inference.py @@ -280,7 +280,7 @@ def __hash__(self): for i in abc_hashable_not_really_hashable: assert not inference.is_hashable(i) - # numpy.array is no longer collections.Hashable as of + # numpy.array is no longer collections.abc.Hashable as of # https://github.com/numpy/numpy/pull/5326, just test # is_hashable() assert not inference.is_hashable(np.array([])) diff --git a/pandas/tests/extension/json/array.py b/pandas/tests/extension/json/array.py index 3e328f545d52f..45574262354f5 100644 --- a/pandas/tests/extension/json/array.py +++ b/pandas/tests/extension/json/array.py @@ -10,7 +10,7 @@ in that case. We *want* the dictionaries to be treated as scalars, so we hack around pandas by using UserDicts. """ -import collections +from collections import UserDict, abc import itertools import numbers import random @@ -21,16 +21,15 @@ from pandas.core.dtypes.base import ExtensionDtype -from pandas import compat from pandas.core.arrays import ExtensionArray class JSONDtype(ExtensionDtype): - type = compat.Mapping + type = abc.Mapping name = 'json' try: - na_value = collections.UserDict() + na_value = UserDict() except AttributeError: # source compatibility with Py2. na_value = {} @@ -78,14 +77,14 @@ def _from_sequence(cls, scalars, dtype=None, copy=False): @classmethod def _from_factorized(cls, values, original): - return cls([collections.UserDict(x) for x in values if x != ()]) + return cls([UserDict(x) for x in values if x != ()]) def __getitem__(self, item): if isinstance(item, numbers.Integral): return self.data[item] elif isinstance(item, np.ndarray) and item.dtype == 'bool': return self._from_sequence([x for x, m in zip(self, item) if m]) - elif isinstance(item, compat.Iterable): + elif isinstance(item, abc.Iterable): # fancy indexing return type(self)([self.data[i] for i in item]) else: @@ -96,8 +95,7 @@ def __setitem__(self, key, value): if isinstance(key, numbers.Integral): self.data[key] = value else: - if not isinstance(value, (type(self), - compat.Sequence)): + if not isinstance(value, (type(self), abc.Sequence)): # broadcast value value = itertools.cycle([value]) @@ -194,6 +192,6 @@ def _values_for_argsort(self): def make_data(): # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer - return [collections.UserDict([ + return [UserDict([ (random.choice(string.ascii_letters), random.randint(0, 100)) for _ in range(random.randint(0, 10))]) for _ in range(100)] diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 8ea5c05125005..ee03e3271d67b 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from collections import OrderedDict +from collections import OrderedDict, abc from datetime import datetime, timedelta import functools import itertools @@ -973,7 +973,7 @@ def test_constructor_sequence_like(self): # GH 3783 # collections.Squence like - class DummyContainer(compat.Sequence): + class DummyContainer(abc.Sequence): def __init__(self, lst): self._lst = lst diff --git a/pandas/tests/frame/test_convert_to.py b/pandas/tests/frame/test_convert_to.py index 43bc925abddba..decd9ec304b37 100644 --- a/pandas/tests/frame/test_convert_to.py +++ b/pandas/tests/frame/test_convert_to.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -import collections -from collections import OrderedDict, defaultdict +from collections import OrderedDict, abc, defaultdict from datetime import datetime import numpy as np @@ -119,7 +118,7 @@ def test_to_records_with_Mapping_type(self): import email from email.parser import Parser - compat.Mapping.register(email.message.Message) + abc.Mapping.register(email.message.Message) headers = Parser().parsestr('From: \n' 'To: \n' @@ -365,10 +364,7 @@ def keys(self): ("B", "