diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index fd2c522dcd0ca..be2a8da58c7e4 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -1,9 +1,13 @@ # Copyright (c) 2012, Lambda Foundry, Inc. # See LICENSE for the license +import bz2 +import gzip +import lzma import os import sys import time import warnings +import zipfile from csv import QUOTE_MINIMAL, QUOTE_NONNUMERIC, QUOTE_NONE from errno import ENOENT @@ -624,16 +628,13 @@ cdef class TextReader: if self.compression: if self.compression == 'gzip': - import gzip if isinstance(source, basestring): source = gzip.GzipFile(source, 'rb') else: source = gzip.GzipFile(fileobj=source) elif self.compression == 'bz2': - import bz2 source = bz2.BZ2File(source, 'rb') elif self.compression == 'zip': - import zipfile zip_file = zipfile.ZipFile(source) zip_names = zip_file.namelist() @@ -648,8 +649,6 @@ cdef class TextReader: raise ValueError('Multiple files found in compressed ' 'zip file %s', str(zip_names)) elif self.compression == 'xz': - lzma = compat.import_lzma() - if isinstance(source, basestring): source = lzma.LZMAFile(source, 'rb') else: diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index 17a75303a617c..be59f838fce9e 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -5,7 +5,6 @@ Cross-compatible functions for Python 2 and 3. Key items to import for 2/3 compatible code: -* iterators: reduce() * lists: lrange(), lmap(), lzip(), lfilter() * iterable method compatibility: iteritems, iterkeys, itervalues * Uses the original method if available, otherwise uses items, keys, values. @@ -100,15 +99,6 @@ def signature(f): 'varargs', 'keywords']) return argspec(args, defaults, varargs, keywords) - def get_range_parameters(data): - """Gets the start, stop, and step parameters from a range object""" - return data.start, data.stop, data.step - - # have to explicitly put builtins into the namespace - intern = sys.intern - reduce = functools.reduce - unichr = chr - # list-producing versions of the major Python iterating functions def lrange(*args, **kwargs): return list(range(*args, **kwargs)) @@ -122,8 +112,6 @@ def lmap(*args, **kwargs): def lfilter(*args, **kwargs): return list(filter(*args, **kwargs)) - from importlib import reload - reload = reload Hashable = collections.abc.Hashable Iterable = collections.abc.Iterable Iterator = collections.abc.Iterator @@ -149,37 +137,12 @@ def bytes_to_str(b, encoding='ascii'): def signature(f): return inspect.getargspec(f) - def get_range_parameters(data): - """Gets the start, stop, and step parameters from a range object""" - # seems we only have indexing ops to infer - # rather than direct accessors - if len(data) > 1: - step = data[1] - data[0] - stop = data[-1] + step - start = data[0] - elif len(data): - start = data[0] - stop = data[0] + 1 - step = 1 - else: - start = stop = 0 - step = 1 - - return start, stop, step - - # import iterator versions of these functions - intern = intern - reduce = reduce - unichr = unichr - # Python 2-builtin ranges produce lists lrange = builtins.range lzip = builtins.zip lmap = builtins.map lfilter = builtins.filter - reload = builtins.reload - Hashable = collections.Hashable Iterable = collections.Iterable Iterator = collections.Iterator @@ -247,7 +210,6 @@ class to receive bound method if PY3: string_types = str, - class_types = type, text_type = str binary_type = bytes @@ -274,11 +236,6 @@ def east_asian_len(data, encoding=None, ambiguous_width=1): else: return len(data) - def import_lzma(): - """ import lzma from the std library """ - import lzma - return lzma - def set_function_name(f, name, cls): """ Bind the name/qualname attributes of the function """ f.__name__ = name @@ -289,7 +246,6 @@ def set_function_name(f, name, cls): return f else: string_types = basestring, - class_types = (type, types.ClassType) text_type = unicode binary_type = str @@ -321,12 +277,6 @@ def east_asian_len(data, encoding=None, ambiguous_width=1): else: return len(data) - def import_lzma(): - """ import the backported lzma library - or raise ImportError if not available """ - from backports import lzma - return lzma - def set_function_name(f, name, cls): """ Bind the name attributes of the function """ f.__name__ = name @@ -335,20 +285,6 @@ def set_function_name(f, name, cls): string_and_binary_types = string_types + (binary_type,) -if PY2: - # In PY2 functools.wraps doesn't provide metadata pytest needs to generate - # decorated tests using parametrization. See pytest GH issue #2782 - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - return wrapper -else: - wraps = functools.wraps - - def add_metaclass(metaclass): """Class decorator for creating a class with a metaclass.""" def wrapper(cls): diff --git a/pandas/conftest.py b/pandas/conftest.py index 3140efa6503e3..1cb518f426299 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -218,8 +218,7 @@ def all_compare_operators(request): return request.param -@pytest.fixture(params=[None, 'gzip', 'bz2', 'zip', - pytest.param('xz', marks=td.skip_if_no_lzma)]) +@pytest.fixture(params=[None, 'gzip', 'bz2', 'zip', 'xz']) def compression(request): """ Fixture for trying common compression types in compression tests @@ -227,8 +226,7 @@ def compression(request): return request.param -@pytest.fixture(params=['gzip', 'bz2', 'zip', - pytest.param('xz', marks=td.skip_if_no_lzma)]) +@pytest.fixture(params=['gzip', 'bz2', 'zip', 'xz']) def compression_only(request): """ Fixture for trying common compression types in compression tests excluding diff --git a/pandas/core/computation/common.py b/pandas/core/computation/common.py index 1e38919affcdd..ef686a1205f44 100644 --- a/pandas/core/computation/common.py +++ b/pandas/core/computation/common.py @@ -1,6 +1,8 @@ +from functools import reduce + import numpy as np -from pandas.compat import reduce, string_types +from pandas.compat import string_types import pandas as pd diff --git a/pandas/core/computation/expr.py b/pandas/core/computation/expr.py index 1d8fea951faab..16795ea8c58e9 100644 --- a/pandas/core/computation/expr.py +++ b/pandas/core/computation/expr.py @@ -2,14 +2,14 @@ """ import ast -from functools import partial +from functools import partial, reduce import itertools as it import operator import tokenize import numpy as np -from pandas.compat import StringIO, lmap, reduce, string_types +from pandas.compat import StringIO, lmap, string_types import pandas as pd from pandas import compat diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 44128c47732d5..ec90e4aa6867c 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -7,7 +7,7 @@ from pandas._libs import index as libindex, lib import pandas.compat as compat -from pandas.compat import get_range_parameters, lrange +from pandas.compat import lrange from pandas.compat.numpy import function as nv from pandas.util._decorators import Appender, cache_readonly @@ -132,7 +132,7 @@ def from_range(cls, data, name=None, dtype=None, **kwargs): '{0}(...) must be called with object coercible to a ' 'range, {1} was passed'.format(cls.__name__, repr(data))) - start, stop, step = get_range_parameters(data) + start, stop, step = data.start, data.stop, data.step return RangeIndex(start, stop, step, dtype=dtype, name=name, **kwargs) @classmethod diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py index d06be8b70fc96..231cca0c6701b 100644 --- a/pandas/core/internals/construction.py +++ b/pandas/core/internals/construction.py @@ -10,8 +10,7 @@ from pandas._libs import lib from pandas._libs.tslibs import IncompatibleFrequency import pandas.compat as compat -from pandas.compat import ( - get_range_parameters, lmap, lrange, raise_with_traceback) +from pandas.compat import lmap, lrange, raise_with_traceback from pandas.core.dtypes.cast import ( construct_1d_arraylike_from_scalar, construct_1d_ndarray_preserving_na, @@ -612,8 +611,7 @@ def sanitize_array(data, index, dtype=None, copy=False, elif isinstance(data, range): # GH#16804 - start, stop, step = get_range_parameters(data) - arr = np.arange(start, stop, step, dtype='int64') + arr = np.arange(data.start, data.stop, data.step, dtype='int64') subarr = _try_cast(arr, False, dtype, copy, raise_cast_failure) else: subarr = _try_cast(data, False, dtype, copy, raise_cast_failure) diff --git a/pandas/io/common.py b/pandas/io/common.py index c1cacf39c5b08..a042a0ae3d7d7 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -1,8 +1,11 @@ """Common IO api utilities""" +import bz2 import codecs from contextlib import closing, contextmanager import csv +import gzip +import lzma import mmap import os import zipfile @@ -364,7 +367,6 @@ def _get_handle(path_or_buf, mode, encoding=None, compression=None, # GZ Compression if compression == 'gzip': - import gzip if is_path: f = gzip.open(path_or_buf, mode) else: @@ -372,7 +374,6 @@ def _get_handle(path_or_buf, mode, encoding=None, compression=None, # BZ Compression elif compression == 'bz2': - import bz2 if is_path: f = bz2.BZ2File(path_or_buf, mode) elif compat.PY2: @@ -404,7 +405,6 @@ def _get_handle(path_or_buf, mode, encoding=None, compression=None, # XZ Compression elif compression == 'xz': - lzma = compat.import_lzma() f = lzma.LZMAFile(path_or_buf, mode) # Unrecognized Compression diff --git a/pandas/io/formats/excel.py b/pandas/io/formats/excel.py index d74722996a660..ec8bc1bbc53dd 100644 --- a/pandas/io/formats/excel.py +++ b/pandas/io/formats/excel.py @@ -1,14 +1,13 @@ """Utilities for conversion to writer-agnostic Excel representation """ +from functools import reduce import itertools import re import warnings import numpy as np -from pandas.compat import reduce - from pandas.core.dtypes import missing from pandas.core.dtypes.common import is_float, is_scalar from pandas.core.dtypes.generic import ABCMultiIndex, ABCPeriodIndex diff --git a/pandas/io/formats/html.py b/pandas/io/formats/html.py index 966656049cf5e..6ba24a95032f8 100644 --- a/pandas/io/formats/html.py +++ b/pandas/io/formats/html.py @@ -8,11 +8,11 @@ from pandas._config import get_option -from pandas.compat import lzip, unichr +from pandas.compat import lzip from pandas.core.dtypes.generic import ABCMultiIndex -from pandas import compat, option_context +from pandas import option_context from pandas.io.common import _is_url from pandas.io.formats.format import TableFormatter, get_level_lengths @@ -145,7 +145,7 @@ def render(self): self._write_table() if self.should_show_dimensions: - by = chr(215) if compat.PY3 else unichr(215) # × + by = chr(215) self.write('

{rows} rows {by} {cols} columns

' .format(rows=len(self.frame), by=by, diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 40a6305bbb42f..f2d5069ccb5d0 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1,4 +1,5 @@ from distutils.version import LooseVersion +from functools import reduce from itertools import product import operator import warnings @@ -7,7 +8,6 @@ from numpy.random import rand, randint, randn import pytest -from pandas.compat import reduce from pandas.errors import PerformanceWarning import pandas.util._test_decorators as td diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index e54da94089cfd..cb5453f20b525 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -19,12 +19,8 @@ @pytest.mark.network -@pytest.mark.parametrize( - "compress_type, extension", [ - ('gzip', '.gz'), ('bz2', '.bz2'), ('zip', '.zip'), - pytest.param('xz', '.xz', marks=td.skip_if_no_lzma) - ] -) +@pytest.mark.parametrize("compress_type, extension", [ + ('gzip', '.gz'), ('bz2', '.bz2'), ('zip', '.zip'), ('xz', '.xz')]) @pytest.mark.parametrize('mode', ['explicit', 'infer']) @pytest.mark.parametrize('engine', ['python', 'c']) def test_compressed_urls(salaries_table, compress_type, extension, mode, diff --git a/pandas/tests/io/test_html.py b/pandas/tests/io/test_html.py index 53471fd5967ce..36b2ac28c067e 100644 --- a/pandas/tests/io/test_html.py +++ b/pandas/tests/io/test_html.py @@ -1,4 +1,5 @@ from functools import partial +from importlib import reload import os import re import threading @@ -7,7 +8,7 @@ from numpy.random import rand import pytest -from pandas.compat import BytesIO, StringIO, is_platform_windows, reload +from pandas.compat import BytesIO, StringIO, is_platform_windows from pandas.errors import ParserError import pandas.util._test_decorators as td diff --git a/pandas/tests/io/test_pickle.py b/pandas/tests/io/test_pickle.py index 8718de4535eb5..c986159acda6a 100644 --- a/pandas/tests/io/test_pickle.py +++ b/pandas/tests/io/test_pickle.py @@ -12,16 +12,19 @@ 3. Move the created pickle to "data/legacy_pickle/" directory. """ +import bz2 from distutils.version import LooseVersion import glob +import gzip +import lzma import os import shutil from warnings import catch_warnings, simplefilter +import zipfile import pytest from pandas.compat import is_platform_little_endian -import pandas.util._test_decorators as td import pandas as pd from pandas import Index @@ -332,18 +335,14 @@ def compress_file(self, src_path, dest_path, compression): return if compression == 'gzip': - import gzip f = gzip.open(dest_path, "w") elif compression == 'bz2': - import bz2 f = bz2.BZ2File(dest_path, "w") elif compression == 'zip': - import zipfile with zipfile.ZipFile(dest_path, "w", compression=zipfile.ZIP_DEFLATED) as f: f.write(src_path, os.path.basename(src_path)) elif compression == 'xz': - lzma = pd.compat.import_lzma() f = lzma.LZMAFile(dest_path, "w") else: msg = 'Unrecognized compression type: {}'.format(compression) @@ -381,10 +380,7 @@ def test_write_explicit_bad(self, compression, get_random_path): df = tm.makeDataFrame() df.to_pickle(path, compression=compression) - @pytest.mark.parametrize('ext', [ - '', '.gz', '.bz2', '.no_compress', - pytest.param('.xz', marks=td.skip_if_no_lzma) - ]) + @pytest.mark.parametrize('ext', ['', '.gz', '.bz2', '.no_compress', '.xz']) def test_write_infer(self, ext, get_random_path): base = get_random_path path1 = base + ext @@ -431,9 +427,7 @@ def test_read_explicit(self, compression, get_random_path): tm.assert_frame_equal(df, df2) @pytest.mark.parametrize('ext', [ - '', '.gz', '.bz2', '.zip', '.no_compress', - pytest.param('.xz', marks=td.skip_if_no_lzma) - ]) + '', '.gz', '.bz2', '.zip', '.no_compress', '.xz']) def test_read_infer(self, ext, get_random_path): base = get_random_path path1 = base + ".raw" diff --git a/pandas/tests/series/test_dtypes.py b/pandas/tests/series/test_dtypes.py index c90437a730631..636c0865a5b68 100644 --- a/pandas/tests/series/test_dtypes.py +++ b/pandas/tests/series/test_dtypes.py @@ -2,6 +2,7 @@ # pylint: disable-msg=E1101,W0612 from datetime import datetime, timedelta +from importlib import reload import string import sys @@ -188,7 +189,7 @@ def test_astype_unicode(self): # Restore the former encoding if former_encoding is not None and former_encoding != "utf-8": - reload(sys) # noqa + reload(sys) sys.setdefaultencoding(former_encoding) @pytest.mark.parametrize("dtype_class", [dict, Series]) diff --git a/pandas/tests/test_compat.py b/pandas/tests/test_compat.py index d5f112e3cb847..c33d62dd87be7 100644 --- a/pandas/tests/test_compat.py +++ b/pandas/tests/test_compat.py @@ -5,11 +5,9 @@ import re -import pytest - from pandas.compat import ( - builtins, get_range_parameters, iteritems, iterkeys, itervalues, lfilter, - lmap, lrange, lzip, re_type) + builtins, iteritems, iterkeys, itervalues, lfilter, lmap, lrange, lzip, + re_type) class TestBuiltinIterators(object): @@ -60,19 +58,5 @@ def test_dict_iterators(self): assert next(iteritems({1: 2})) == (1, 2) -class TestCompatFunctions(object): - - @pytest.mark.parametrize( - 'start,stop,step', [(0, 10, 2), (11, -2, -1), (0, -5, 1), (2, 4, 8)]) - def test_get_range_parameters(self, start, stop, step): - rng = range(start, stop, step) - start_expected, stop_expected, step_expected = start, stop, step - - start_result, stop_result, step_result = get_range_parameters(rng) - assert start_result == start_expected - assert stop_result == stop_expected - assert step_result == step_expected - - def test_re_type(): assert isinstance(re.compile(''), re_type) diff --git a/pandas/util/_test_decorators.py b/pandas/util/_test_decorators.py index 0331661c3131f..c0a972388d886 100644 --- a/pandas/util/_test_decorators.py +++ b/pandas/util/_test_decorators.py @@ -28,8 +28,7 @@ def test_foo(): import pytest -from pandas.compat import ( - PY3, import_lzma, is_platform_32bit, is_platform_windows) +from pandas.compat import PY3, is_platform_32bit, is_platform_windows from pandas.compat.numpy import _np_version_under1p15 from pandas.core.computation.expressions import ( @@ -110,13 +109,6 @@ def _skip_if_no_scipy(): safe_import('scipy.signal')) -def _skip_if_no_lzma(): - try: - import_lzma() - except ImportError: - return True - - def skip_if_no(package, min_version=None): """ Generic function to help skip test functions when required packages are not @@ -176,8 +168,6 @@ def decorated_func(func): lang=locale.getlocale()[0])) skip_if_no_scipy = pytest.mark.skipif(_skip_if_no_scipy(), reason="Missing SciPy requirement") -skip_if_no_lzma = pytest.mark.skipif(_skip_if_no_lzma(), - reason="need backports.lzma to run") skip_if_no_ne = pytest.mark.skipif(not _USE_NUMEXPR, reason="numexpr enabled->{enabled}, " "installed->{installed}".format( diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 3ef235688e3aa..2664e60ce8616 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -1,15 +1,18 @@ +import bz2 from collections import Counter from contextlib import contextmanager from datetime import datetime from functools import wraps +import gzip +import lzma import os import re from shutil import rmtree import string -import sys import tempfile import traceback import warnings +import zipfile import numpy as np from numpy.random import rand, randn @@ -20,8 +23,7 @@ from pandas._libs import testing as _testing import pandas.compat as compat from pandas.compat import ( - PY2, PY3, httplib, lmap, lrange, lzip, raise_with_traceback, string_types, - unichr) + PY2, PY3, httplib, lmap, lrange, lzip, raise_with_traceback, string_types) from pandas.core.dtypes.common import ( is_bool, is_categorical_dtype, is_datetime64_dtype, is_datetime64tz_dtype, @@ -177,16 +179,12 @@ def decompress_file(path, compression): if compression is None: f = open(path, 'rb') elif compression == 'gzip': - import gzip f = gzip.open(path, 'rb') elif compression == 'bz2': - import bz2 f = bz2.BZ2File(path, 'rb') elif compression == 'xz': - lzma = compat.import_lzma() f = lzma.LZMAFile(path, 'rb') elif compression == 'zip': - import zipfile zip_file = zipfile.ZipFile(path) zip_names = zip_file.namelist() if len(zip_names) == 1: @@ -236,7 +234,7 @@ def write_to_compressed(compression, path, data, dest="test"): import bz2 compress_method = bz2.BZ2File elif compression == "xz": - lzma = compat.import_lzma() + import lzma compress_method = lzma.LZMAFile else: msg = "Unrecognized compression type: {}".format(compression) @@ -367,7 +365,7 @@ def randbool(size=(), p=0.5): RANDS_CHARS = np.array(list(string.ascii_letters + string.digits), dtype=(np.str_, 1)) -RANDU_CHARS = np.array(list("".join(map(unichr, lrange(1488, 1488 + 26))) + +RANDU_CHARS = np.array(list("".join(map(chr, lrange(1488, 1488 + 26))) + string.digits), dtype=(np.unicode_, 1)) @@ -421,32 +419,6 @@ def close(fignum=None): _close(fignum) -# ----------------------------------------------------------------------------- -# Stdout / stderr decorators - - -@contextmanager -def set_defaultencoding(encoding): - """ - Set default encoding (as given by sys.getdefaultencoding()) to the given - encoding; restore on exit. - - Parameters - ---------- - encoding : str - """ - if not PY2: - raise ValueError("set_defaultencoding context is only available " - "in Python 2.") - orig = sys.getdefaultencoding() - reload(sys) # noqa:F821 - sys.setdefaultencoding(encoding) - try: - yield - finally: - sys.setdefaultencoding(orig) - - # ----------------------------------------------------------------------------- # contextmanager to ensure the file cleanup @@ -2207,7 +2179,7 @@ def network(t, url="http://www.google.com", from pytest import skip t.network = True - @compat.wraps(t) + @wraps(t) def wrapper(*args, **kwargs): if check_before_test and not raise_on_error: if not can_connect(url, error_classes):