From dcce74494a7e9c699acf90019e72e95339131c10 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 11:22:34 -0400 Subject: [PATCH 1/8] Remove dateutil version check tests --- .../tests/indexes/datetimes/test_timezones.py | 13 +++------ pandas/tests/indexes/datetimes/test_tools.py | 9 ------ .../tests/scalar/timestamp/test_rendering.py | 11 ++------ .../tests/scalar/timestamp/test_timezones.py | 28 ++++++------------- pandas/tests/tseries/offsets/test_offsets.py | 20 ------------- 5 files changed, 15 insertions(+), 66 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index 368dc68e516df..908d563eca8fa 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -2,7 +2,6 @@ Tests for DatetimeIndex timezone-related methods """ from datetime import date, datetime, time, timedelta, tzinfo -from distutils.version import LooseVersion import dateutil from dateutil.tz import gettz, tzlocal @@ -554,14 +553,10 @@ def test_dti_construction_ambiguous_endpoint(self, tz): assert times[0] == Timestamp('2013-10-26 23:00', tz=tz, freq="H") if str(tz).startswith('dateutil'): - if LooseVersion(dateutil.__version__) < LooseVersion('2.6.0'): - # see GH#14621 - assert times[-1] == Timestamp('2013-10-27 01:00:00+0000', - tz=tz, freq="H") - elif LooseVersion(dateutil.__version__) > LooseVersion('2.6.0'): - # fixed ambiguous behavior - assert times[-1] == Timestamp('2013-10-27 01:00:00+0100', - tz=tz, freq="H") + # fixed ambiguous behavior + # see GH#14621 + assert times[-1] == Timestamp('2013-10-27 01:00:00+0100', + tz=tz, freq="H") else: assert times[-1] == Timestamp('2013-10-27 01:00:00+0000', tz=tz, freq="H") diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 2a5ae92cb59f5..a971a1088860a 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -2,10 +2,8 @@ import calendar from datetime import datetime, time -from distutils.version import LooseVersion import locale -import dateutil from dateutil.parser import parse from dateutil.tz.tz import tzoffset import numpy as np @@ -1739,8 +1737,6 @@ def test_parsers_dayfirst_yearfirst(self, cache): # 2.5.2 20/12/21 [dayfirst=1, yearfirst=0] -> 2021-12-20 00:00:00 # 2.5.3 20/12/21 [dayfirst=1, yearfirst=0] -> 2021-12-20 00:00:00 - is_lt_253 = LooseVersion(dateutil.__version__) < LooseVersion('2.5.3') - # str : dayfirst, yearfirst, expected cases = {'10-11-12': [(False, False, datetime(2012, 10, 11)), @@ -1762,11 +1758,6 @@ def test_parsers_dayfirst_yearfirst(self, cache): for date_str, values in cases.items(): for dayfirst, yearfirst, expected in values: - # odd comparisons across version - # let's just skip - if dayfirst and yearfirst and is_lt_253: - continue - # compare with dateutil result dateutil_result = parse(date_str, dayfirst=dayfirst, yearfirst=yearfirst) diff --git a/pandas/tests/scalar/timestamp/test_rendering.py b/pandas/tests/scalar/timestamp/test_rendering.py index c16ab39d642d9..69ea0a810c4ce 100644 --- a/pandas/tests/scalar/timestamp/test_rendering.py +++ b/pandas/tests/scalar/timestamp/test_rendering.py @@ -1,7 +1,5 @@ -from distutils.version import LooseVersion import pprint -import dateutil import pytest import pytz # noqa # a test below uses pytz but only inside a `eval` call @@ -10,13 +8,8 @@ class TestTimestampRendering: - # dateutil zone change (only matters for repr) - if LooseVersion(dateutil.__version__) >= LooseVersion('2.6.0'): - timezones = ['UTC', 'Asia/Tokyo', 'US/Eastern', - 'dateutil/US/Pacific'] - else: - timezones = ['UTC', 'Asia/Tokyo', 'US/Eastern', - 'dateutil/America/Los_Angeles'] + timezones = ['UTC', 'Asia/Tokyo', 'US/Eastern', + 'dateutil/US/Pacific'] @pytest.mark.parametrize('tz', timezones) @pytest.mark.parametrize('freq', ['D', 'M', 'S', 'N']) diff --git a/pandas/tests/scalar/timestamp/test_timezones.py b/pandas/tests/scalar/timestamp/test_timezones.py index f67ecdaf746f6..914423fcf5ba7 100644 --- a/pandas/tests/scalar/timestamp/test_timezones.py +++ b/pandas/tests/scalar/timestamp/test_timezones.py @@ -2,7 +2,6 @@ Tests for Timestamp timezone-related methods """ from datetime import date, datetime, timedelta -from distutils.version import LooseVersion import dateutil from dateutil.tz import gettz, tzoffset @@ -145,18 +144,11 @@ def test_tz_localize_ambiguous_compat(self): assert result_pytz.value == result_dateutil.value assert result_pytz.value == 1382835600000000000 - if LooseVersion(dateutil.__version__) < LooseVersion('2.6.0'): - # dateutil 2.6 buggy w.r.t. ambiguous=0 - # see gh-14621 - # see https://github.com/dateutil/dateutil/issues/321 - assert (result_pytz.to_pydatetime().tzname() == - result_dateutil.to_pydatetime().tzname()) - assert str(result_pytz) == str(result_dateutil) - elif LooseVersion(dateutil.__version__) > LooseVersion('2.6.0'): - # fixed ambiguous behavior - assert result_pytz.to_pydatetime().tzname() == 'GMT' - assert result_dateutil.to_pydatetime().tzname() == 'BST' - assert str(result_pytz) != str(result_dateutil) + # fixed ambiguous behavior + # see gh-14621 + assert result_pytz.to_pydatetime().tzname() == 'GMT' + assert result_dateutil.to_pydatetime().tzname() == 'BST' + assert str(result_pytz) != str(result_dateutil) # 1 hour difference result_pytz = naive.tz_localize(pytz_zone, ambiguous=1) @@ -164,12 +156,10 @@ def test_tz_localize_ambiguous_compat(self): assert result_pytz.value == result_dateutil.value assert result_pytz.value == 1382832000000000000 - # dateutil < 2.6 is buggy w.r.t. ambiguous timezones - if LooseVersion(dateutil.__version__) > LooseVersion('2.5.3'): - # see gh-14621 - assert str(result_pytz) == str(result_dateutil) - assert (result_pytz.to_pydatetime().tzname() == - result_dateutil.to_pydatetime().tzname()) + # see gh-14621 + assert str(result_pytz) == str(result_dateutil) + assert (result_pytz.to_pydatetime().tzname() == + result_dateutil.to_pydatetime().tzname()) @pytest.mark.parametrize('tz', [pytz.timezone('US/Eastern'), gettz('US/Eastern'), diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 151cd2a42ecef..5683924ee1283 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -1,5 +1,4 @@ from datetime import date, datetime, timedelta -from distutils.version import LooseVersion import numpy as np import pytest @@ -2998,25 +2997,6 @@ def _make_timestamp(self, string, hrs_offset, tz): offset_string = '-{hrs:02d}00'.format(hrs=-1 * hrs_offset) return Timestamp(string + offset_string).tz_convert(tz) - def test_fallback_plural(self): - # test moving from daylight savings to standard time - import dateutil - for tz, utc_offsets in self.timezone_utc_offsets.items(): - hrs_pre = utc_offsets['utc_offset_daylight'] - hrs_post = utc_offsets['utc_offset_standard'] - - if LooseVersion(dateutil.__version__) < LooseVersion('2.6.0'): - # buggy ambiguous behavior in 2.6.0 - # GH 14621 - # https://github.com/dateutil/dateutil/issues/321 - self._test_all_offsets( - n=3, tstart=self._make_timestamp(self.ts_pre_fallback, - hrs_pre, tz), - expected_utc_offset=hrs_post) - elif LooseVersion(dateutil.__version__) > LooseVersion('2.6.0'): - # fixed, but skip the test - continue - def test_springforward_plural(self): # test moving from standard to daylight savings for tz, utc_offsets in self.timezone_utc_offsets.items(): From 1dc8845895d64b968c47155cb145432649fba466 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 15:09:52 -0400 Subject: [PATCH 2/8] Remove more dead version checking code --- pandas/io/html.py | 13 +++++++++---- pandas/io/pytables.py | 5 ----- pandas/tests/io/conftest.py | 5 ----- pandas/tests/io/excel/test_style.py | 11 ++--------- pandas/tests/io/pytables/test_pytables.py | 3 +-- pandas/tests/io/test_parquet.py | 8 ++------ 6 files changed, 14 insertions(+), 31 deletions(-) diff --git a/pandas/io/html.py b/pandas/io/html.py index 15b9d25f6be6c..9f21f480817f4 100644 --- a/pandas/io/html.py +++ b/pandas/io/html.py @@ -10,7 +10,8 @@ import re from pandas.compat import raise_with_traceback -from pandas.compat._optional import import_optional_dependency +from pandas.compat._optional import ( + VERSIONS, import_optional_dependency, version_message) from pandas.errors import AbstractMethodError, EmptyDataError from pandas.core.dtypes.common import is_list_like @@ -831,9 +832,13 @@ def _parser_dispatch(flavor): raise ImportError( "BeautifulSoup4 (bs4) not found, please install it") import bs4 - if LooseVersion(bs4.__version__) <= LooseVersion('4.2.0'): - raise ValueError("A minimum version of BeautifulSoup 4.2.1 " - "is required") + minimum_bs4_version = VERSIONS['bs4'] + if LooseVersion(bs4.__version__) <= LooseVersion(minimum_bs4_version): + raise ImportError(version_message.format( + minimum_bs4_version, + 'bs4', + bs4.__version__ + )) else: if not _HAS_LXML: diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 97d5b1dd2a1e5..c8c27f62cef34 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -5,7 +5,6 @@ import copy from datetime import date, datetime -from distutils.version import LooseVersion import itertools import os import re @@ -227,10 +226,6 @@ def _tables(): import tables _table_mod = tables - # version requirements - if LooseVersion(tables.__version__) < LooseVersion('3.0.0'): - raise ImportError("PyTables version >= 3.0.0 is required") - # set the file open policy # return the file open policy; this changes as of pytables 3.1 # depending on the HDF5 version diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index a4e778a68c728..08a46a131c393 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -45,11 +45,6 @@ def s3_resource(tips_file, jsonl_file): boto3 = pytest.importorskip('boto3') botocore = pytest.importorskip('botocore') - if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"): - # botocore leaks an uncatchable ResourceWarning before 1.11.0; - # see GH 23731 and https://github.com/boto/botocore/issues/1464 - pytest.skip("botocore is leaking resources before 1.11.0") - with tm.ensure_safe_environment_variables(): # temporary workaround as moto fails for botocore >= 1.11 otherwise, # see https://github.com/spulec/moto/issues/1924 & 1952 diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index d8426f54bb188..a7f591483a6cb 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -107,15 +107,8 @@ def custom_converter(css): assert cell1.font.color.rgb != cell2.font.color.rgb assert cell2.font.color.rgb == alpha + '0000FF' elif ref == 'D4': - # This fails with engine=xlsxwriter due to - # https://bitbucket.org/openpyxl/openpyxl/issues/800 - if engine == 'xlsxwriter' \ - and (LooseVersion(openpyxl.__version__) < - LooseVersion('2.4.6')): - pass - else: - assert cell1.font.underline != cell2.font.underline - assert cell2.font.underline == 'single' + assert cell1.font.underline != cell2.font.underline + assert cell2.font.underline == 'single' elif ref == 'B5': assert not cell1.border.left.style assert (cell2.border.top.style == diff --git a/pandas/tests/io/pytables/test_pytables.py b/pandas/tests/io/pytables/test_pytables.py index be318ede2df4a..40cc05c317471 100644 --- a/pandas/tests/io/pytables/test_pytables.py +++ b/pandas/tests/io/pytables/test_pytables.py @@ -43,8 +43,7 @@ 'release beyong 3.4.4 to support numpy 1.16x')) -_default_compressor = ('blosc' if LooseVersion(tables.__version__) >= - LooseVersion('2.2') else 'zlib') +_default_compressor = 'blosc' ignore_natural_naming_warning = pytest.mark.filterwarnings( diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index db5c92fb681a2..6f33713b3b76e 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -454,10 +454,8 @@ class TestParquetFastParquet(Base): def test_basic(self, fp, df_full): df = df_full - # additional supported types for fastparquet - if LooseVersion(fastparquet.__version__) >= LooseVersion('0.1.4'): - df['datetime_tz'] = pd.date_range('20130101', periods=3, - tz='US/Eastern') + df['datetime_tz'] = pd.date_range('20130101', periods=3, + tz='US/Eastern') df['timedelta'] = pd.timedelta_range('1 day', periods=3) check_round_trip(df, fp) @@ -485,8 +483,6 @@ def test_unsupported(self, fp): self.check_error_on_write(df, fp, ValueError) def test_categorical(self, fp): - if LooseVersion(fastparquet.__version__) < LooseVersion("0.1.3"): - pytest.skip("CategoricalDtype not supported for older fp") df = pd.DataFrame({'a': pd.Categorical(list('abc'))}) check_round_trip(df, fp) From 32fcd00baaf13b9b1f354a3aabdb9266d1dcb14b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 15:17:49 -0400 Subject: [PATCH 3/8] Flake8 and fix bs4 comparison --- pandas/io/html.py | 2 +- pandas/tests/io/conftest.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/io/html.py b/pandas/io/html.py index 9f21f480817f4..8c662a14aed3f 100644 --- a/pandas/io/html.py +++ b/pandas/io/html.py @@ -833,7 +833,7 @@ def _parser_dispatch(flavor): "BeautifulSoup4 (bs4) not found, please install it") import bs4 minimum_bs4_version = VERSIONS['bs4'] - if LooseVersion(bs4.__version__) <= LooseVersion(minimum_bs4_version): + if LooseVersion(bs4.__version__) < LooseVersion(minimum_bs4_version): raise ImportError(version_message.format( minimum_bs4_version, 'bs4', diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 08a46a131c393..ebe9517ab0c4d 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -43,7 +43,6 @@ def s3_resource(tips_file, jsonl_file): """ pytest.importorskip('s3fs') boto3 = pytest.importorskip('boto3') - botocore = pytest.importorskip('botocore') with tm.ensure_safe_environment_variables(): # temporary workaround as moto fails for botocore >= 1.11 otherwise, From 124b2c5de7e576b3fdb7c8d5cce6c9e7b938a35e Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 15:33:56 -0400 Subject: [PATCH 4/8] Use import_optional_dependency again --- pandas/io/html.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pandas/io/html.py b/pandas/io/html.py index 8c662a14aed3f..d84ce77303d6c 100644 --- a/pandas/io/html.py +++ b/pandas/io/html.py @@ -10,8 +10,7 @@ import re from pandas.compat import raise_with_traceback -from pandas.compat._optional import ( - VERSIONS, import_optional_dependency, version_message) +from pandas.compat._optional import import_optional_dependency from pandas.errors import AbstractMethodError, EmptyDataError from pandas.core.dtypes.common import is_list_like @@ -831,14 +830,8 @@ def _parser_dispatch(flavor): if not _HAS_BS4: raise ImportError( "BeautifulSoup4 (bs4) not found, please install it") - import bs4 - minimum_bs4_version = VERSIONS['bs4'] - if LooseVersion(bs4.__version__) < LooseVersion(minimum_bs4_version): - raise ImportError(version_message.format( - minimum_bs4_version, - 'bs4', - bs4.__version__ - )) + # Although we call this above, we want to raise before use. + bs4 = import_optional_dependency('bs4') else: if not _HAS_LXML: From 5b8a54f5a416b26ec6405129fe2c6416bfc0ce83 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 15:34:23 -0400 Subject: [PATCH 5/8] Remove unused import --- pandas/io/html.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/html.py b/pandas/io/html.py index d84ce77303d6c..26e4c2f5ad72c 100644 --- a/pandas/io/html.py +++ b/pandas/io/html.py @@ -4,7 +4,6 @@ """ from collections import abc -from distutils.version import LooseVersion import numbers import os import re From 345cf3f5df25de76d7b2d2042d914f099a62e696 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 16:56:35 -0400 Subject: [PATCH 6/8] Change bs4 import test to check for ImportError and add whatsnew --- doc/source/whatsnew/v0.25.0.rst | 1 + pandas/tests/io/test_html.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 901e4f6942897..cfd8b7884033f 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -499,6 +499,7 @@ Other API Changes - Most Pandas classes had a ``__bytes__`` method, which was used for getting a python2-style bytestring representation of the object. This method has been removed as a part of dropping Python2 (:issue:`26447`) - The ``.str``-accessor has been disabled for 1-level :class:`MultiIndex`, use :meth:`MultiIndex.to_flat_index` if necessary (:issue:`23679`) - Removed support of gtk package for clipboards (:issue:`26563`) +- Using an unsupported version of Beautiful Soup 4 will now raise an ``ImportError`` instead of a ``ValueError`` (:issue:`27063`) .. _whatsnew_0250.deprecations: diff --git a/pandas/tests/io/test_html.py b/pandas/tests/io/test_html.py index bd6fc6f57c496..9f9fcabbfe42c 100644 --- a/pandas/tests/io/test_html.py +++ b/pandas/tests/io/test_html.py @@ -54,7 +54,7 @@ def assert_framelist_equal(list1, list2, *args, **kwargs): def test_bs4_version_fails(monkeypatch, datapath): import bs4 monkeypatch.setattr(bs4, '__version__', '4.2') - with pytest.raises(ValueError, match="minimum version"): + with pytest.raises(ImportError, match="Pandas requires version"): read_html(datapath("io", "data", "spam.html"), flavor='bs4') From de1d276f968e518d7de33178a6879bbf94b4afa6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 26 Jun 2019 17:05:38 -0400 Subject: [PATCH 7/8] Some linting --- pandas/tests/io/conftest.py | 1 - pandas/tests/io/excel/test_style.py | 2 -- pandas/tests/io/test_parquet.py | 1 - 3 files changed, 4 deletions(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index ebe9517ab0c4d..d431e835a07ce 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,4 +1,3 @@ -from distutils.version import LooseVersion import os import pytest diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index a7f591483a6cb..d8971777f6eb4 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -1,5 +1,3 @@ -from distutils.version import LooseVersion - import numpy as np import pytest diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index 6f33713b3b76e..a90412df16336 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -1,6 +1,5 @@ """ test parquet compat """ import datetime -from distutils.version import LooseVersion import os from warnings import catch_warnings From 33259db7d82022f07982a065009080ce05112ead Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 27 Jun 2019 16:54:52 -0500 Subject: [PATCH 8/8] noqa necessary imports --- pandas/io/html.py | 4 ++-- pandas/tests/io/test_parquet.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/io/html.py b/pandas/io/html.py index 26e4c2f5ad72c..d54489aabf1ed 100644 --- a/pandas/io/html.py +++ b/pandas/io/html.py @@ -829,8 +829,8 @@ def _parser_dispatch(flavor): if not _HAS_BS4: raise ImportError( "BeautifulSoup4 (bs4) not found, please install it") - # Although we call this above, we want to raise before use. - bs4 = import_optional_dependency('bs4') + # Although we call this above, we want to raise here right before use. + bs4 = import_optional_dependency('bs4') # noqa:F841 else: if not _HAS_LXML: diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index a90412df16336..f5f8dac71d095 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -507,7 +507,7 @@ def test_partition_cols_supported(self, fp, df_full): df.to_parquet(path, engine="fastparquet", partition_cols=partition_cols, compression=None) assert os.path.exists(path) - import fastparquet + import fastparquet # noqa: F811 actual_partition_cols = fastparquet.ParquetFile(path, False).cats assert len(actual_partition_cols) == 2 @@ -519,7 +519,7 @@ def test_partition_on_supported(self, fp, df_full): df.to_parquet(path, engine="fastparquet", compression=None, partition_on=partition_cols) assert os.path.exists(path) - import fastparquet + import fastparquet # noqa: F811 actual_partition_cols = fastparquet.ParquetFile(path, False).cats assert len(actual_partition_cols) == 2