From 092b3b93335665a1a96dd47d6c2bc13319f901f0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 14:25:13 -0800 Subject: [PATCH 01/26] Deprecate time_rule --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/tests/tseries/offsets/test_offsets.py | 8 ++++++++ pandas/tseries/offsets.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 0b2b526dfe9e7..3ac2ba44a880d 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1137,6 +1137,7 @@ Deprecations - :func:`pandas.types.is_datetimetz` is deprecated in favor of `pandas.types.is_datetime64tz` (:issue:`23917`) - Creating a :class:`TimedeltaIndex` or :class:`DatetimeIndex` by passing range arguments `start`, `end`, and `periods` is deprecated in favor of :func:`timedelta_range` and :func:`date_range` (:issue:`23919`) - Passing a string alias like ``'datetime64[ns, UTC]'`` as the `unit` parameter to :class:`DatetimeTZDtype` is deprecated. Use :class:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`). +- Passing a `time_rule` to `pandas.tseries.offsets.generate_range` is deprecated and will raise a ``TypeError`` in a future version. Pass an ``offset`` instead (:issue:`????`) .. _whatsnew_0240.deprecations.datetimelike_int_ops: diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 030887ac731f3..deec01882658b 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -49,6 +49,14 @@ class WeekDay(object): #### +def test_time_rule_deprecated(): + start = datetime(2007, 10, 1) + end = datetime(2012, 4, 9) + + with tm.assert_produces_warning(FutureWarning): + offsets.generate_range(start=start, end=end, time_rule="9W") + + def test_to_m8(): valb = datetime(2007, 10, 1) valu = _to_m8(valb) diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 45f10a2f06fa2..9b08ab5412136 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2,6 +2,7 @@ from datetime import date, datetime, timedelta import functools import operator +import warnings from dateutil.easter import easter import numpy as np @@ -2487,6 +2488,9 @@ def generate_range(start=None, end=None, periods=None, """ if time_rule is not None: + warnings.warn("`time_rule` is deprecated and will be removed in a " + "future version. Use `offset` instead.", + FutureWarning) from pandas.tseries.frequencies import get_offset offset = get_offset(time_rule) From fbcc04bbf8156ac8b14e72090d867f7c6d230fda Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 14:40:41 -0800 Subject: [PATCH 02/26] cleanup --- pandas/tests/tseries/offsets/test_offsets.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index deec01882658b..95db05a97f2cd 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -61,17 +61,11 @@ def test_to_m8(): valb = datetime(2007, 10, 1) valu = _to_m8(valb) assert isinstance(valu, np.datetime64) - # assert valu == np.datetime64(datetime(2007,10,1)) - # def test_datetime64_box(): - # valu = np.datetime64(datetime(2007,10,1)) - # valb = _dt_box(valu) - # assert type(valb) == datetime - # assert valb == datetime(2007,10,1) - ##### - # DateOffset Tests - ##### +##### +# DateOffset Tests +##### class Base(object): From ac52857a9766d82f0a5a55beb88a789c1b1a39bd Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 14:53:33 -0800 Subject: [PATCH 03/26] fix test --- pandas/tests/tseries/offsets/test_offsets.py | 4 +++- pandas/tseries/offsets.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 95db05a97f2cd..9ac21d3587dc9 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -54,7 +54,9 @@ def test_time_rule_deprecated(): end = datetime(2012, 4, 9) with tm.assert_produces_warning(FutureWarning): - offsets.generate_range(start=start, end=end, time_rule="9W") + # Note: generate_range returns a generator, and the warning is not + # issued until the first __next__ call + list(offsets.generate_range(start=start, end=end, time_rule="W")) def test_to_m8(): diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 9b08ab5412136..107b9e8f186d8 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2490,7 +2490,7 @@ def generate_range(start=None, end=None, periods=None, if time_rule is not None: warnings.warn("`time_rule` is deprecated and will be removed in a " "future version. Use `offset` instead.", - FutureWarning) + FutureWarning, stacklevel=2) from pandas.tseries.frequencies import get_offset offset = get_offset(time_rule) From 121c3732ac0d20b21d5dd454b3c52726e3377bb8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 14:53:50 -0800 Subject: [PATCH 04/26] Fix indexing with ellipsis, GH#21282 --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/core/arrays/datetimelike.py | 4 ++++ pandas/tests/indexes/datetimes/test_indexing.py | 9 +++++++++ pandas/tests/indexes/period/test_indexing.py | 8 ++++++++ pandas/tests/indexes/timedeltas/test_indexing.py | 8 ++++++++ 5 files changed, 30 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 3ac2ba44a880d..59995decd9873 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1419,6 +1419,7 @@ Indexing - Bug in :func:`Index.union` and :func:`Index.intersection` where name of the ``Index`` of the result was not computed correctly for certain cases (:issue:`9943`, :issue:`9862`) - Bug in :class:`Index` slicing with boolean :class:`Index` may raise ``TypeError`` (:issue:`22533`) - Bug in ``PeriodArray.__setitem__`` when accepting slice and list-like value (:issue:`23978`) +- Bug in :class:`DatetimeIndex`, :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would lose their ``freq`` attribute (:issue:`21282`) Missing ^^^^^^^ diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 45eec41e498d1..aa7c2a837a1ab 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -327,6 +327,10 @@ def __getitem__(self, key): "numpy.newaxis (`None`) and integer or boolean " "arrays are valid indices") + if key is Ellipsis: + # GH#21282 + return self.copy() + getitem = self._data.__getitem__ if is_int: val = getitem(key) diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 944c925dabe3e..c3b00133228d8 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -16,6 +16,15 @@ class TestGetItem(object): + def test_ellipsis(self): + # GH#21282 + idx = pd.date_range('2011-01-01', '2011-01-31', freq='D', + tz='Asia/Tokyo', name='idx') + + result = idx[...] + assert result.equals(idx) + assert result is not idx + def test_getitem(self): idx1 = pd.date_range('2011-01-01', '2011-01-31', freq='D', name='idx') idx2 = pd.date_range('2011-01-01', '2011-01-31', freq='D', diff --git a/pandas/tests/indexes/period/test_indexing.py b/pandas/tests/indexes/period/test_indexing.py index aaa1126e92f3d..29b96604b7ea8 100644 --- a/pandas/tests/indexes/period/test_indexing.py +++ b/pandas/tests/indexes/period/test_indexing.py @@ -13,6 +13,14 @@ class TestGetItem(object): + def test_ellipsis(self): + # GH#21282 + idx = period_range('2011-01-01', '2011-01-31', freq='D', + name='idx') + + result = idx[...] + assert result.equals(idx) + assert result is not idx def test_getitem(self): idx1 = pd.period_range('2011-01-01', '2011-01-31', freq='D', diff --git a/pandas/tests/indexes/timedeltas/test_indexing.py b/pandas/tests/indexes/timedeltas/test_indexing.py index 94d694b644eb8..4e98732456d2c 100644 --- a/pandas/tests/indexes/timedeltas/test_indexing.py +++ b/pandas/tests/indexes/timedeltas/test_indexing.py @@ -9,6 +9,14 @@ class TestGetItem(object): + def test_ellipsis(self): + # GH#21282 + idx = timedelta_range('1 day', '31 day', freq='D', name='idx') + + result = idx[...] + assert result.equals(idx) + assert result is not idx + def test_getitem(self): idx1 = timedelta_range('1 day', '31 day', freq='D', name='idx') From ef66bb96ef6450d3b98cd9b749f6e62642c92d0b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 15:12:17 -0800 Subject: [PATCH 05/26] isort tests.arithmetic, tests.dtypes --- pandas/tests/arithmetic/conftest.py | 7 ++- pandas/tests/arithmetic/test_datetime64.py | 18 ++++---- pandas/tests/arithmetic/test_numeric.py | 11 +++-- pandas/tests/arithmetic/test_object.py | 8 ++-- pandas/tests/arithmetic/test_period.py | 9 ++-- pandas/tests/arithmetic/test_timedelta64.py | 13 +++--- pandas/tests/dtypes/test_cast.py | 32 ++++++-------- pandas/tests/dtypes/test_common.py | 13 +++--- pandas/tests/dtypes/test_concat.py | 4 +- pandas/tests/dtypes/test_dtypes.py | 27 ++++++------ pandas/tests/dtypes/test_generic.py | 5 ++- pandas/tests/dtypes/test_inference.py | 48 +++++++++------------ pandas/tests/dtypes/test_missing.py | 25 +++++------ pandas/tests/internals/test_internals.py | 31 ++++++------- setup.cfg | 14 ------ 15 files changed, 117 insertions(+), 148 deletions(-) diff --git a/pandas/tests/arithmetic/conftest.py b/pandas/tests/arithmetic/conftest.py index 44e6cc664de6d..15b5f0b8a9db0 100644 --- a/pandas/tests/arithmetic/conftest.py +++ b/pandas/tests/arithmetic/conftest.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- -import pytest - import numpy as np -import pandas as pd +import pytest from pandas.compat import long -import pandas.util.testing as tm +import pandas as pd +import pandas.util.testing as tm # ------------------------------------------------------------------ # Helper Functions diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 02e9c212b56ef..e71934a44165d 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2,27 +2,25 @@ # Arithmetic tests for DataFrame/Series/Index/Array classes that should # behave identically. # Specifically for datetime64 and datetime64tz dtypes -import operator from datetime import datetime, timedelta -import warnings from itertools import product, starmap +import operator +import warnings import numpy as np import pytest import pytz -import pandas as pd -import pandas.util.testing as tm - -from pandas.compat.numpy import np_datetime64_compat -from pandas.errors import PerformanceWarning, NullFrequencyError - from pandas._libs.tslibs.conversion import localize_pydatetime from pandas._libs.tslibs.offsets import shift_months +from pandas.compat.numpy import np_datetime64_compat +from pandas.errors import NullFrequencyError, PerformanceWarning +import pandas as pd from pandas import ( - Timestamp, Timedelta, Period, Series, date_range, NaT, - DatetimeIndex, TimedeltaIndex) + DatetimeIndex, NaT, Period, Series, Timedelta, TimedeltaIndex, Timestamp, + date_range) +import pandas.util.testing as tm def assert_all(obj): diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 2d26959c65632..56ccfd205331c 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -5,16 +5,15 @@ from decimal import Decimal import operator -import pytest import numpy as np - -import pandas as pd -import pandas.util.testing as tm +import pytest from pandas.compat import PY3, Iterable -from pandas.core import ops -from pandas import Timedelta, Series, Index, TimedeltaIndex +import pandas as pd +from pandas import Index, Series, Timedelta, TimedeltaIndex +from pandas.core import ops +import pandas.util.testing as tm # ------------------------------------------------------------------ # Comparisons diff --git a/pandas/tests/arithmetic/test_object.py b/pandas/tests/arithmetic/test_object.py index da5055cc7b737..d6f737d58cc09 100644 --- a/pandas/tests/arithmetic/test_object.py +++ b/pandas/tests/arithmetic/test_object.py @@ -4,15 +4,13 @@ # Specifically for object dtype import operator -import pytest import numpy as np +import pytest import pandas as pd -import pandas.util.testing as tm -from pandas.core import ops - from pandas import Series, Timestamp - +from pandas.core import ops +import pandas.util.testing as tm # ------------------------------------------------------------------ # Comparisons diff --git a/pandas/tests/arithmetic/test_period.py b/pandas/tests/arithmetic/test_period.py index 7158eae376ba6..0ff79f9d9a0d3 100644 --- a/pandas/tests/arithmetic/test_period.py +++ b/pandas/tests/arithmetic/test_period.py @@ -7,16 +7,15 @@ import numpy as np import pytest -import pandas as pd -import pandas.util.testing as tm - from pandas._libs.tslibs.period import IncompatibleFrequency from pandas.errors import PerformanceWarning +import pandas as pd +from pandas import Period, PeriodIndex, Series, period_range from pandas.core import ops -from pandas import Period, PeriodIndex, period_range, Series -from pandas.tseries.frequencies import to_offset +import pandas.util.testing as tm +from pandas.tseries.frequencies import to_offset # ------------------------------------------------------------------ # Comparisons diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 5404d3f5f1915..1fc1411d5a752 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -3,17 +3,16 @@ # behave identically. from datetime import datetime, timedelta -import pytest import numpy as np - -import pandas as pd -import pandas.util.testing as tm +import pytest from pandas.errors import NullFrequencyError, PerformanceWarning + +import pandas as pd from pandas import ( - timedelta_range, - Timedelta, Timestamp, NaT, Series, TimedeltaIndex, DatetimeIndex, - DataFrame) + DataFrame, DatetimeIndex, NaT, Series, Timedelta, TimedeltaIndex, + Timestamp, timedelta_range) +import pandas.util.testing as tm def get_upcast_box(box, vector): diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index fcdcf96098f16..871e71ea2e4b0 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -5,30 +5,24 @@ """ -import pytest -from datetime import datetime, timedelta, date -import numpy as np +from datetime import date, datetime, timedelta -import pandas as pd -from pandas import (Timedelta, Timestamp, DatetimeIndex, - DataFrame, NaT, Period, Series) +import numpy as np +import pytest from pandas.core.dtypes.cast import ( - maybe_downcast_to_dtype, - maybe_convert_objects, - cast_scalar_to_array, - infer_dtype_from_scalar, - infer_dtype_from_array, - find_common_type, - construct_1d_object_array_from_listlike, + cast_scalar_to_array, construct_1d_arraylike_from_scalar, construct_1d_ndarray_preserving_na, - construct_1d_arraylike_from_scalar) + construct_1d_object_array_from_listlike, find_common_type, + infer_dtype_from_array, infer_dtype_from_scalar, maybe_convert_objects, + maybe_downcast_to_dtype) +from pandas.core.dtypes.common import is_dtype_equal from pandas.core.dtypes.dtypes import ( - CategoricalDtype, - DatetimeTZDtype, - PeriodDtype) -from pandas.core.dtypes.common import ( - is_dtype_equal) + CategoricalDtype, DatetimeTZDtype, PeriodDtype) + +import pandas as pd +from pandas import ( + DataFrame, DatetimeIndex, NaT, Period, Series, Timedelta, Timestamp) from pandas.util import testing as tm diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index e176d273b916c..52eac65bffd82 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- -import pytest import numpy as np -import pandas as pd +import pytest -from pandas.core.dtypes.dtypes import (DatetimeTZDtype, PeriodDtype, - CategoricalDtype, IntervalDtype) -from pandas.core.sparse.api import SparseDtype +import pandas.util._test_decorators as td import pandas.core.dtypes.common as com -import pandas.util._test_decorators as td +from pandas.core.dtypes.dtypes import ( + CategoricalDtype, DatetimeTZDtype, IntervalDtype, PeriodDtype) + +import pandas as pd +from pandas.core.sparse.api import SparseDtype import pandas.util.testing as tm diff --git a/pandas/tests/dtypes/test_concat.py b/pandas/tests/dtypes/test_concat.py index 35623415571c0..d58f8ee3b74f1 100644 --- a/pandas/tests/dtypes/test_concat.py +++ b/pandas/tests/dtypes/test_concat.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- import pytest + import pandas.core.dtypes.concat as _concat + from pandas import ( - Index, DatetimeIndex, PeriodIndex, TimedeltaIndex, Series, Period) + DatetimeIndex, Index, Period, PeriodIndex, Series, TimedeltaIndex) @pytest.mark.parametrize('to_concat, expected', [ diff --git a/pandas/tests/dtypes/test_dtypes.py b/pandas/tests/dtypes/test_dtypes.py index 81d08ac71bf6d..1ba5051ecc0f4 100644 --- a/pandas/tests/dtypes/test_dtypes.py +++ b/pandas/tests/dtypes/test_dtypes.py @@ -1,24 +1,21 @@ # -*- coding: utf-8 -*- import re -import pytest import numpy as np -import pandas as pd -from pandas import ( - Series, Categorical, CategoricalIndex, IntervalIndex, date_range) +import pytest -from pandas.core.dtypes.dtypes import ( - DatetimeTZDtype, PeriodDtype, - IntervalDtype, CategoricalDtype, registry, _pandas_registry) from pandas.core.dtypes.common import ( - is_categorical_dtype, is_categorical, - is_datetime64tz_dtype, is_datetimetz, - is_period_dtype, is_period, - is_dtype_equal, is_datetime64_ns_dtype, - is_datetime64_dtype, is_interval_dtype, - is_datetime64_any_dtype, is_string_dtype, - is_bool_dtype, -) + is_bool_dtype, is_categorical, is_categorical_dtype, + is_datetime64_any_dtype, is_datetime64_dtype, is_datetime64_ns_dtype, + is_datetime64tz_dtype, is_datetimetz, is_dtype_equal, is_interval_dtype, + is_period, is_period_dtype, is_string_dtype) +from pandas.core.dtypes.dtypes import ( + CategoricalDtype, DatetimeTZDtype, IntervalDtype, PeriodDtype, + _pandas_registry, registry) + +import pandas as pd +from pandas import ( + Categorical, CategoricalIndex, IntervalIndex, Series, date_range) from pandas.core.sparse.api import SparseDtype import pandas.util.testing as tm diff --git a/pandas/tests/dtypes/test_generic.py b/pandas/tests/dtypes/test_generic.py index 53fa482bdeaef..96f92fccc5a71 100644 --- a/pandas/tests/dtypes/test_generic.py +++ b/pandas/tests/dtypes/test_generic.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- from warnings import catch_warnings, simplefilter + import numpy as np -import pandas as pd + from pandas.core.dtypes import generic as gt + +import pandas as pd from pandas.util import testing as tm diff --git a/pandas/tests/dtypes/test_inference.py b/pandas/tests/dtypes/test_inference.py index 0c22b595bc74d..d470e12212274 100644 --- a/pandas/tests/dtypes/test_inference.py +++ b/pandas/tests/dtypes/test_inference.py @@ -5,42 +5,34 @@ related to inference and not otherwise tested in types/test_common.py """ -from warnings import catch_warnings, simplefilter import collections -import re -from datetime import datetime, date, timedelta, time +from datetime import date, datetime, time, timedelta from decimal import Decimal -from numbers import Number from fractions import Fraction +from numbers import Number +import re +from warnings import catch_warnings, simplefilter + import numpy as np -import pytz import pytest -import pandas as pd -from pandas._libs import lib, iNaT, missing as libmissing -from pandas import (Series, Index, DataFrame, Timedelta, - DatetimeIndex, TimedeltaIndex, Timestamp, - Panel, Period, Categorical, isna, Interval, - DateOffset) -from pandas import compat -from pandas.compat import u, PY2, StringIO, lrange +import pytz + +from pandas._libs import iNaT, lib, missing as libmissing +from pandas.compat import PY2, StringIO, lrange, u +import pandas.util._test_decorators as td + from pandas.core.dtypes import inference from pandas.core.dtypes.common import ( - is_timedelta64_dtype, - is_timedelta64_ns_dtype, - is_datetime64_dtype, - is_datetime64_ns_dtype, - is_datetime64_any_dtype, - is_datetime64tz_dtype, - is_number, - is_integer, - is_float, - is_bool, - is_scalar, - is_scipy_sparse, - ensure_int32, - ensure_categorical) + ensure_categorical, ensure_int32, is_bool, is_datetime64_any_dtype, + is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype, + is_float, is_integer, is_number, is_scalar, is_scipy_sparse, + is_timedelta64_dtype, is_timedelta64_ns_dtype) + +import pandas as pd +from pandas import ( + Categorical, DataFrame, DateOffset, DatetimeIndex, Index, Interval, Panel, + Period, Series, Timedelta, TimedeltaIndex, Timestamp, compat, isna) from pandas.util import testing as tm -import pandas.util._test_decorators as td @pytest.fixture(params=[True, False], ids=str) diff --git a/pandas/tests/dtypes/test_missing.py b/pandas/tests/dtypes/test_missing.py index cb3f5933c885f..56c9395d0f802 100644 --- a/pandas/tests/dtypes/test_missing.py +++ b/pandas/tests/dtypes/test_missing.py @@ -1,25 +1,26 @@ # -*- coding: utf-8 -*- -import pytest -from warnings import catch_warnings, simplefilter -import numpy as np from datetime import datetime -from pandas.util import testing as tm +from warnings import catch_warnings, simplefilter -import pandas as pd -from pandas.core import config as cf -from pandas.compat import u +import numpy as np +import pytest from pandas._libs import missing as libmissing from pandas._libs.tslib import iNaT -from pandas import (NaT, Float64Index, Series, - DatetimeIndex, TimedeltaIndex, date_range) +from pandas.compat import u + from pandas.core.dtypes.common import is_scalar from pandas.core.dtypes.dtypes import ( - DatetimeTZDtype, PeriodDtype, IntervalDtype) + DatetimeTZDtype, IntervalDtype, PeriodDtype) from pandas.core.dtypes.missing import ( - array_equivalent, isna, notna, isnull, notnull, - na_value_for_dtype) + array_equivalent, isna, isnull, na_value_for_dtype, notna, notnull) + +import pandas as pd +from pandas import ( + DatetimeIndex, Float64Index, NaT, Series, TimedeltaIndex, date_range) +from pandas.core import config as cf +from pandas.util import testing as tm @pytest.mark.parametrize('notna_f', [notna, notnull]) diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index 26cd39c4b807c..b811f6c7813d9 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -1,27 +1,28 @@ # -*- coding: utf-8 -*- # pylint: disable=W0102 -from datetime import datetime, date +from datetime import date, datetime +from distutils.version import LooseVersion +import itertools import operator +import re import sys -import pytest + import numpy as np +import pytest -import re -from distutils.version import LooseVersion -import itertools -from pandas import (Index, MultiIndex, DataFrame, DatetimeIndex, - Series, Categorical, TimedeltaIndex, SparseArray) -from pandas.compat import OrderedDict, lrange -from pandas.core.internals import (SingleBlockManager, - make_block, BlockManager) +from pandas._libs.internals import BlockPlacement +from pandas.compat import OrderedDict, lrange, u, zip + +import pandas as pd +from pandas import ( + Categorical, DataFrame, DatetimeIndex, Index, MultiIndex, Series, + SparseArray, TimedeltaIndex) import pandas.core.algorithms as algos +from pandas.core.internals import BlockManager, SingleBlockManager, make_block import pandas.util.testing as tm -import pandas as pd -from pandas._libs.internals import BlockPlacement -from pandas.util.testing import (assert_almost_equal, assert_frame_equal, - randn, assert_series_equal) -from pandas.compat import zip, u +from pandas.util.testing import ( + assert_almost_equal, assert_frame_equal, assert_series_equal, randn) # in 3.6.1 a c-api slicing function changed, see src/compat_helper.h PY361 = LooseVersion(sys.version) >= LooseVersion('3.6.1') diff --git a/setup.cfg b/setup.cfg index 44df79d1b60d2..0590aa90f5438 100644 --- a/setup.cfg +++ b/setup.cfg @@ -251,20 +251,6 @@ skip= pandas/tests/api/test_types.py, pandas/tests/api/test_api.py, pandas/tests/tools/test_numeric.py, - pandas/tests/dtypes/test_concat.py, - pandas/tests/dtypes/test_generic.py, - pandas/tests/dtypes/test_common.py, - pandas/tests/dtypes/test_cast.py, - pandas/tests/dtypes/test_dtypes.py, - pandas/tests/dtypes/test_inference.py, - pandas/tests/dtypes/test_missing.py, - pandas/tests/arithmetic/test_numeric.py, - pandas/tests/arithmetic/test_object.py, - pandas/tests/arithmetic/test_period.py, - pandas/tests/arithmetic/test_datetime64.py, - pandas/tests/arithmetic/conftest.py, - pandas/tests/arithmetic/test_timedelta64.py, - pandas/tests/internals/test_internals.py, pandas/tests/groupby/test_value_counts.py, pandas/tests/groupby/test_filters.py, pandas/tests/groupby/test_nth.py, From 91738d3ec521c61aaf86e0c549fc2172ff29d22e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 15:34:56 -0800 Subject: [PATCH 06/26] raise on date_range with NaT for either start or end --- pandas/core/arrays/datetimes.py | 2 ++ pandas/tests/indexes/datetimes/test_date_range.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index b74ede4547249..9fd2d642ffc05 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -264,6 +264,8 @@ def _generate_range(cls, start, end, periods, freq, tz=None, if closed is not None: raise ValueError("Closed has to be None if not both of start" "and end are defined") + if start is NaT or end is NaT: + raise ValueError("Neither `start` nor `end` can be NaT") left_closed, right_closed = dtl.validate_endpoints(closed) diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index 54a04ab6f80fd..bc37e3423df9f 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -80,6 +80,13 @@ def test_date_range_timestamp_equiv_preserve_frequency(self): class TestDateRanges(TestData): + def test_date_range_nat(self): + # GH#11587 + with pytest.raises(ValueError): + date_range(start='2016-01-01', end=pd.NaT, freq='D') + with pytest.raises(ValueError): + date_range(start=pd.NaT, end='2016-01-01', freq='D') + def test_date_range_out_of_bounds(self): # GH#14187 with pytest.raises(OutOfBoundsDatetime): @@ -533,12 +540,14 @@ class TestGenRangeGeneration(object): def test_generate(self): rng1 = list(generate_range(START, END, offset=BDay())) - rng2 = list(generate_range(START, END, time_rule='B')) + with tm.assert_produces_warning(FutureWarning): + rng2 = list(generate_range(START, END, time_rule='B')) assert rng1 == rng2 def test_generate_cday(self): rng1 = list(generate_range(START, END, offset=CDay())) - rng2 = list(generate_range(START, END, time_rule='C')) + with tm.assert_produces_warning(FutureWarning): + rng2 = list(generate_range(START, END, time_rule='C')) assert rng1 == rng2 def test_1(self): From bb0d06516021b6d6209668166562b84f7c270c53 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 15:35:31 -0800 Subject: [PATCH 07/26] revert noncentral changes --- pandas/tests/arithmetic/conftest.py | 7 +-- pandas/tests/arithmetic/test_datetime64.py | 18 ++++---- pandas/tests/arithmetic/test_numeric.py | 11 ++--- pandas/tests/arithmetic/test_object.py | 8 ++-- pandas/tests/arithmetic/test_period.py | 9 ++-- pandas/tests/arithmetic/test_timedelta64.py | 13 +++--- pandas/tests/dtypes/test_cast.py | 32 ++++++++------ pandas/tests/dtypes/test_common.py | 13 +++--- pandas/tests/dtypes/test_concat.py | 4 +- pandas/tests/dtypes/test_dtypes.py | 27 ++++++------ pandas/tests/dtypes/test_generic.py | 5 +-- pandas/tests/dtypes/test_inference.py | 48 ++++++++++++--------- pandas/tests/dtypes/test_missing.py | 25 ++++++----- pandas/tests/internals/test_internals.py | 31 +++++++------ setup.cfg | 14 ++++++ 15 files changed, 148 insertions(+), 117 deletions(-) diff --git a/pandas/tests/arithmetic/conftest.py b/pandas/tests/arithmetic/conftest.py index 15b5f0b8a9db0..44e6cc664de6d 100644 --- a/pandas/tests/arithmetic/conftest.py +++ b/pandas/tests/arithmetic/conftest.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- -import numpy as np import pytest -from pandas.compat import long - +import numpy as np import pandas as pd + +from pandas.compat import long import pandas.util.testing as tm + # ------------------------------------------------------------------ # Helper Functions diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index e71934a44165d..02e9c212b56ef 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2,25 +2,27 @@ # Arithmetic tests for DataFrame/Series/Index/Array classes that should # behave identically. # Specifically for datetime64 and datetime64tz dtypes -from datetime import datetime, timedelta -from itertools import product, starmap import operator +from datetime import datetime, timedelta import warnings +from itertools import product, starmap import numpy as np import pytest import pytz +import pandas as pd +import pandas.util.testing as tm + +from pandas.compat.numpy import np_datetime64_compat +from pandas.errors import PerformanceWarning, NullFrequencyError + from pandas._libs.tslibs.conversion import localize_pydatetime from pandas._libs.tslibs.offsets import shift_months -from pandas.compat.numpy import np_datetime64_compat -from pandas.errors import NullFrequencyError, PerformanceWarning -import pandas as pd from pandas import ( - DatetimeIndex, NaT, Period, Series, Timedelta, TimedeltaIndex, Timestamp, - date_range) -import pandas.util.testing as tm + Timestamp, Timedelta, Period, Series, date_range, NaT, + DatetimeIndex, TimedeltaIndex) def assert_all(obj): diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 56ccfd205331c..2d26959c65632 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -5,16 +5,17 @@ from decimal import Decimal import operator -import numpy as np import pytest - -from pandas.compat import PY3, Iterable +import numpy as np import pandas as pd -from pandas import Index, Series, Timedelta, TimedeltaIndex -from pandas.core import ops import pandas.util.testing as tm +from pandas.compat import PY3, Iterable +from pandas.core import ops +from pandas import Timedelta, Series, Index, TimedeltaIndex + + # ------------------------------------------------------------------ # Comparisons diff --git a/pandas/tests/arithmetic/test_object.py b/pandas/tests/arithmetic/test_object.py index d6f737d58cc09..da5055cc7b737 100644 --- a/pandas/tests/arithmetic/test_object.py +++ b/pandas/tests/arithmetic/test_object.py @@ -4,13 +4,15 @@ # Specifically for object dtype import operator -import numpy as np import pytest +import numpy as np import pandas as pd -from pandas import Series, Timestamp -from pandas.core import ops import pandas.util.testing as tm +from pandas.core import ops + +from pandas import Series, Timestamp + # ------------------------------------------------------------------ # Comparisons diff --git a/pandas/tests/arithmetic/test_period.py b/pandas/tests/arithmetic/test_period.py index 0ff79f9d9a0d3..7158eae376ba6 100644 --- a/pandas/tests/arithmetic/test_period.py +++ b/pandas/tests/arithmetic/test_period.py @@ -7,16 +7,17 @@ import numpy as np import pytest +import pandas as pd +import pandas.util.testing as tm + from pandas._libs.tslibs.period import IncompatibleFrequency from pandas.errors import PerformanceWarning -import pandas as pd -from pandas import Period, PeriodIndex, Series, period_range from pandas.core import ops -import pandas.util.testing as tm - +from pandas import Period, PeriodIndex, period_range, Series from pandas.tseries.frequencies import to_offset + # ------------------------------------------------------------------ # Comparisons diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 1fc1411d5a752..5404d3f5f1915 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -3,17 +3,18 @@ # behave identically. from datetime import datetime, timedelta -import numpy as np import pytest - -from pandas.errors import NullFrequencyError, PerformanceWarning +import numpy as np import pandas as pd -from pandas import ( - DataFrame, DatetimeIndex, NaT, Series, Timedelta, TimedeltaIndex, - Timestamp, timedelta_range) import pandas.util.testing as tm +from pandas.errors import NullFrequencyError, PerformanceWarning +from pandas import ( + timedelta_range, + Timedelta, Timestamp, NaT, Series, TimedeltaIndex, DatetimeIndex, + DataFrame) + def get_upcast_box(box, vector): """ diff --git a/pandas/tests/dtypes/test_cast.py b/pandas/tests/dtypes/test_cast.py index 871e71ea2e4b0..fcdcf96098f16 100644 --- a/pandas/tests/dtypes/test_cast.py +++ b/pandas/tests/dtypes/test_cast.py @@ -5,24 +5,30 @@ """ -from datetime import date, datetime, timedelta - -import numpy as np import pytest +from datetime import datetime, timedelta, date +import numpy as np + +import pandas as pd +from pandas import (Timedelta, Timestamp, DatetimeIndex, + DataFrame, NaT, Period, Series) from pandas.core.dtypes.cast import ( - cast_scalar_to_array, construct_1d_arraylike_from_scalar, + maybe_downcast_to_dtype, + maybe_convert_objects, + cast_scalar_to_array, + infer_dtype_from_scalar, + infer_dtype_from_array, + find_common_type, + construct_1d_object_array_from_listlike, construct_1d_ndarray_preserving_na, - construct_1d_object_array_from_listlike, find_common_type, - infer_dtype_from_array, infer_dtype_from_scalar, maybe_convert_objects, - maybe_downcast_to_dtype) -from pandas.core.dtypes.common import is_dtype_equal + construct_1d_arraylike_from_scalar) from pandas.core.dtypes.dtypes import ( - CategoricalDtype, DatetimeTZDtype, PeriodDtype) - -import pandas as pd -from pandas import ( - DataFrame, DatetimeIndex, NaT, Period, Series, Timedelta, Timestamp) + CategoricalDtype, + DatetimeTZDtype, + PeriodDtype) +from pandas.core.dtypes.common import ( + is_dtype_equal) from pandas.util import testing as tm diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index 52eac65bffd82..e176d273b916c 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- -import numpy as np import pytest +import numpy as np +import pandas as pd -import pandas.util._test_decorators as td +from pandas.core.dtypes.dtypes import (DatetimeTZDtype, PeriodDtype, + CategoricalDtype, IntervalDtype) +from pandas.core.sparse.api import SparseDtype import pandas.core.dtypes.common as com -from pandas.core.dtypes.dtypes import ( - CategoricalDtype, DatetimeTZDtype, IntervalDtype, PeriodDtype) - -import pandas as pd -from pandas.core.sparse.api import SparseDtype +import pandas.util._test_decorators as td import pandas.util.testing as tm diff --git a/pandas/tests/dtypes/test_concat.py b/pandas/tests/dtypes/test_concat.py index d58f8ee3b74f1..35623415571c0 100644 --- a/pandas/tests/dtypes/test_concat.py +++ b/pandas/tests/dtypes/test_concat.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- import pytest - import pandas.core.dtypes.concat as _concat - from pandas import ( - DatetimeIndex, Index, Period, PeriodIndex, Series, TimedeltaIndex) + Index, DatetimeIndex, PeriodIndex, TimedeltaIndex, Series, Period) @pytest.mark.parametrize('to_concat, expected', [ diff --git a/pandas/tests/dtypes/test_dtypes.py b/pandas/tests/dtypes/test_dtypes.py index 1ba5051ecc0f4..81d08ac71bf6d 100644 --- a/pandas/tests/dtypes/test_dtypes.py +++ b/pandas/tests/dtypes/test_dtypes.py @@ -1,21 +1,24 @@ # -*- coding: utf-8 -*- import re - -import numpy as np import pytest -from pandas.core.dtypes.common import ( - is_bool_dtype, is_categorical, is_categorical_dtype, - is_datetime64_any_dtype, is_datetime64_dtype, is_datetime64_ns_dtype, - is_datetime64tz_dtype, is_datetimetz, is_dtype_equal, is_interval_dtype, - is_period, is_period_dtype, is_string_dtype) -from pandas.core.dtypes.dtypes import ( - CategoricalDtype, DatetimeTZDtype, IntervalDtype, PeriodDtype, - _pandas_registry, registry) - +import numpy as np import pandas as pd from pandas import ( - Categorical, CategoricalIndex, IntervalIndex, Series, date_range) + Series, Categorical, CategoricalIndex, IntervalIndex, date_range) + +from pandas.core.dtypes.dtypes import ( + DatetimeTZDtype, PeriodDtype, + IntervalDtype, CategoricalDtype, registry, _pandas_registry) +from pandas.core.dtypes.common import ( + is_categorical_dtype, is_categorical, + is_datetime64tz_dtype, is_datetimetz, + is_period_dtype, is_period, + is_dtype_equal, is_datetime64_ns_dtype, + is_datetime64_dtype, is_interval_dtype, + is_datetime64_any_dtype, is_string_dtype, + is_bool_dtype, +) from pandas.core.sparse.api import SparseDtype import pandas.util.testing as tm diff --git a/pandas/tests/dtypes/test_generic.py b/pandas/tests/dtypes/test_generic.py index 96f92fccc5a71..53fa482bdeaef 100644 --- a/pandas/tests/dtypes/test_generic.py +++ b/pandas/tests/dtypes/test_generic.py @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- from warnings import catch_warnings, simplefilter - import numpy as np - -from pandas.core.dtypes import generic as gt - import pandas as pd +from pandas.core.dtypes import generic as gt from pandas.util import testing as tm diff --git a/pandas/tests/dtypes/test_inference.py b/pandas/tests/dtypes/test_inference.py index d470e12212274..0c22b595bc74d 100644 --- a/pandas/tests/dtypes/test_inference.py +++ b/pandas/tests/dtypes/test_inference.py @@ -5,34 +5,42 @@ related to inference and not otherwise tested in types/test_common.py """ +from warnings import catch_warnings, simplefilter import collections -from datetime import date, datetime, time, timedelta +import re +from datetime import datetime, date, timedelta, time from decimal import Decimal -from fractions import Fraction from numbers import Number -import re -from warnings import catch_warnings, simplefilter - +from fractions import Fraction import numpy as np -import pytest import pytz - -from pandas._libs import iNaT, lib, missing as libmissing -from pandas.compat import PY2, StringIO, lrange, u -import pandas.util._test_decorators as td - +import pytest +import pandas as pd +from pandas._libs import lib, iNaT, missing as libmissing +from pandas import (Series, Index, DataFrame, Timedelta, + DatetimeIndex, TimedeltaIndex, Timestamp, + Panel, Period, Categorical, isna, Interval, + DateOffset) +from pandas import compat +from pandas.compat import u, PY2, StringIO, lrange from pandas.core.dtypes import inference from pandas.core.dtypes.common import ( - ensure_categorical, ensure_int32, is_bool, is_datetime64_any_dtype, - is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype, - is_float, is_integer, is_number, is_scalar, is_scipy_sparse, - is_timedelta64_dtype, is_timedelta64_ns_dtype) - -import pandas as pd -from pandas import ( - Categorical, DataFrame, DateOffset, DatetimeIndex, Index, Interval, Panel, - Period, Series, Timedelta, TimedeltaIndex, Timestamp, compat, isna) + is_timedelta64_dtype, + is_timedelta64_ns_dtype, + is_datetime64_dtype, + is_datetime64_ns_dtype, + is_datetime64_any_dtype, + is_datetime64tz_dtype, + is_number, + is_integer, + is_float, + is_bool, + is_scalar, + is_scipy_sparse, + ensure_int32, + ensure_categorical) from pandas.util import testing as tm +import pandas.util._test_decorators as td @pytest.fixture(params=[True, False], ids=str) diff --git a/pandas/tests/dtypes/test_missing.py b/pandas/tests/dtypes/test_missing.py index 56c9395d0f802..cb3f5933c885f 100644 --- a/pandas/tests/dtypes/test_missing.py +++ b/pandas/tests/dtypes/test_missing.py @@ -1,26 +1,25 @@ # -*- coding: utf-8 -*- -from datetime import datetime +import pytest from warnings import catch_warnings, simplefilter - import numpy as np -import pytest +from datetime import datetime +from pandas.util import testing as tm -from pandas._libs import missing as libmissing -from pandas._libs.tslib import iNaT +import pandas as pd +from pandas.core import config as cf from pandas.compat import u +from pandas._libs import missing as libmissing +from pandas._libs.tslib import iNaT +from pandas import (NaT, Float64Index, Series, + DatetimeIndex, TimedeltaIndex, date_range) from pandas.core.dtypes.common import is_scalar from pandas.core.dtypes.dtypes import ( - DatetimeTZDtype, IntervalDtype, PeriodDtype) + DatetimeTZDtype, PeriodDtype, IntervalDtype) from pandas.core.dtypes.missing import ( - array_equivalent, isna, isnull, na_value_for_dtype, notna, notnull) - -import pandas as pd -from pandas import ( - DatetimeIndex, Float64Index, NaT, Series, TimedeltaIndex, date_range) -from pandas.core import config as cf -from pandas.util import testing as tm + array_equivalent, isna, notna, isnull, notnull, + na_value_for_dtype) @pytest.mark.parametrize('notna_f', [notna, notnull]) diff --git a/pandas/tests/internals/test_internals.py b/pandas/tests/internals/test_internals.py index b811f6c7813d9..26cd39c4b807c 100644 --- a/pandas/tests/internals/test_internals.py +++ b/pandas/tests/internals/test_internals.py @@ -1,28 +1,27 @@ # -*- coding: utf-8 -*- # pylint: disable=W0102 -from datetime import date, datetime -from distutils.version import LooseVersion -import itertools +from datetime import datetime, date import operator -import re import sys - -import numpy as np import pytest +import numpy as np -from pandas._libs.internals import BlockPlacement -from pandas.compat import OrderedDict, lrange, u, zip - -import pandas as pd -from pandas import ( - Categorical, DataFrame, DatetimeIndex, Index, MultiIndex, Series, - SparseArray, TimedeltaIndex) +import re +from distutils.version import LooseVersion +import itertools +from pandas import (Index, MultiIndex, DataFrame, DatetimeIndex, + Series, Categorical, TimedeltaIndex, SparseArray) +from pandas.compat import OrderedDict, lrange +from pandas.core.internals import (SingleBlockManager, + make_block, BlockManager) import pandas.core.algorithms as algos -from pandas.core.internals import BlockManager, SingleBlockManager, make_block import pandas.util.testing as tm -from pandas.util.testing import ( - assert_almost_equal, assert_frame_equal, assert_series_equal, randn) +import pandas as pd +from pandas._libs.internals import BlockPlacement +from pandas.util.testing import (assert_almost_equal, assert_frame_equal, + randn, assert_series_equal) +from pandas.compat import zip, u # in 3.6.1 a c-api slicing function changed, see src/compat_helper.h PY361 = LooseVersion(sys.version) >= LooseVersion('3.6.1') diff --git a/setup.cfg b/setup.cfg index 0590aa90f5438..44df79d1b60d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -251,6 +251,20 @@ skip= pandas/tests/api/test_types.py, pandas/tests/api/test_api.py, pandas/tests/tools/test_numeric.py, + pandas/tests/dtypes/test_concat.py, + pandas/tests/dtypes/test_generic.py, + pandas/tests/dtypes/test_common.py, + pandas/tests/dtypes/test_cast.py, + pandas/tests/dtypes/test_dtypes.py, + pandas/tests/dtypes/test_inference.py, + pandas/tests/dtypes/test_missing.py, + pandas/tests/arithmetic/test_numeric.py, + pandas/tests/arithmetic/test_object.py, + pandas/tests/arithmetic/test_period.py, + pandas/tests/arithmetic/test_datetime64.py, + pandas/tests/arithmetic/conftest.py, + pandas/tests/arithmetic/test_timedelta64.py, + pandas/tests/internals/test_internals.py, pandas/tests/groupby/test_value_counts.py, pandas/tests/groupby/test_filters.py, pandas/tests/groupby/test_nth.py, From ff3a5c075e0bc627d74361b5ecb3c6cd65750923 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 15:43:19 -0800 Subject: [PATCH 08/26] more useful error messages on validate_frequency failure --- pandas/core/arrays/datetimelike.py | 10 +++++++--- pandas/tests/indexes/datetimes/test_construction.py | 11 +++++++++++ pandas/tests/indexes/datetimes/test_date_range.py | 5 +++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index aa7c2a837a1ab..b43e449697708 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -551,9 +551,13 @@ def _validate_frequency(cls, index, freq, **kwargs): if index.size == 0 or inferred == freq.freqstr: return None - on_freq = cls._generate_range(start=index[0], end=None, - periods=len(index), freq=freq, **kwargs) - if not np.array_equal(index.asi8, on_freq.asi8): + try: + on_freq = cls._generate_range(start=index[0], end=None, + periods=len(index), freq=freq, + **kwargs) + assert np.array_equal(index.asi8, on_freq.asi8) + except (ValueError, AssertionError): + # GH#11587 if index[0] is NaT _generate_range will raise ValueError raise ValueError('Inferred frequency {infer} from passed values ' 'does not conform to passed frequency {passed}' .format(infer=inferred, passed=freq.freqstr)) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 5de79044bc239..06e760be0692c 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -20,6 +20,17 @@ class TestDatetimeIndex(object): + def test_freq_validation_with_nat(self): + # GH#11587 make sure we get a useful error message when generate_range + # raises + msg = ("Inferred frequency None from passed values does not conform " + "to passed frequency D") + with pytest.raises(ValueError, match=msg): + DatetimeIndex([pd.NaT, pd.Timestamp('2011-01-01')], freq='D') + with pytest.raises(ValueError, match=msg): + DatetimeIndex([pd.NaT, pd.Timestamp('2011-01-01').value], + freq='D') + def test_dti_with_period_data_raises(self): # GH#23675 data = pd.PeriodIndex(['2016Q1', '2016Q2'], freq='Q') diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index bc37e3423df9f..fed7aeb83b483 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -82,9 +82,10 @@ def test_date_range_timestamp_equiv_preserve_frequency(self): class TestDateRanges(TestData): def test_date_range_nat(self): # GH#11587 - with pytest.raises(ValueError): + msg = "Neither `start` nor `end` can be NaT" + with pytest.raises(ValueError, match=msg): date_range(start='2016-01-01', end=pd.NaT, freq='D') - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=msg): date_range(start=pd.NaT, end='2016-01-01', freq='D') def test_date_range_out_of_bounds(self): From e54159d1a3f2e9ceff04d9b54a4748d09fb73c66 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 15:58:58 -0800 Subject: [PATCH 09/26] Fix DTI-->Categorical -->DTI roundtrip --- pandas/core/arrays/datetimes.py | 17 ++++++++++--- .../indexes/datetimes/test_construction.py | 24 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 9fd2d642ffc05..ae30e473c9279 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -14,9 +14,9 @@ from pandas.util._decorators import Appender from pandas.core.dtypes.common import ( - _INT64_DTYPE, _NS_DTYPE, is_datetime64_dtype, is_datetime64tz_dtype, - is_extension_type, is_float_dtype, is_int64_dtype, is_object_dtype, - is_period_dtype, is_string_dtype, is_timedelta64_dtype) + _INT64_DTYPE, _NS_DTYPE, is_categorical_dtype, is_datetime64_dtype, + is_datetime64tz_dtype, is_extension_type, is_float_dtype, is_int64_dtype, + is_object_dtype, is_period_dtype, is_string_dtype, is_timedelta64_dtype) from pandas.core.dtypes.dtypes import DatetimeTZDtype from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries from pandas.core.dtypes.missing import isna @@ -1649,6 +1649,17 @@ def maybe_convert_dtype(data, copy): raise TypeError("Passing PeriodDtype data is invalid. " "Use `data.to_timestamp()` instead") + elif is_categorical_dtype(data): + if isna(data).any(): + # slow-path + data = list(data) + data = np.array(data, dtype=np.object_) + copy = False + else: + # TODO: does this always make a copy? If so, set copy=False + # GH#18664 preserve tz in going DTI->Categorical->DTI + data = data.categories[data.codes] + elif is_extension_type(data) and not is_datetime64tz_dtype(data): # Includes categorical # TODO: We have no tests for these diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 06e760be0692c..d12162c099379 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -31,6 +31,30 @@ def test_freq_validation_with_nat(self): DatetimeIndex([pd.NaT, pd.Timestamp('2011-01-01').value], freq='D') + def test_categorical_preserves_tz(self): + # GH#18664 retain tz when going DTI-->Categorical-->DTI + + dti = pd.DatetimeIndex( + [pd.NaT, '2015-01-01', '1999-04-06 15:14:13', '2015-01-01'], + tz='US/Eastern') + ci = pd.CategoricalIndex(dti) + carr = pd.Categorical(dti) + cser = pd.Series(ci) + + for obj in [ci, carr, cser]: + result = pd.DatetimeIndex(obj) + tm.assert_index_equal(result, dti) + + # no-NaT case has a fastpath + dti2 = dti[1:] + ci2 = pd.CategoricalIndex(dti2) + carr2 = pd.Categorical(dti2) + cser2 = pd.Series(ci2) + + for obj in [ci2, carr2, cser2]: + result = pd.DatetimeIndex(obj) + tm.assert_index_equal(result, dti2) + def test_dti_with_period_data_raises(self): # GH#23675 data = pd.PeriodIndex(['2016Q1', '2016Q2'], freq='Q') From 44e01269c35ddb0a215c73202d765cc67888f806 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 16:03:40 -0800 Subject: [PATCH 10/26] extend test to DTA --- .../tests/indexes/datetimes/test_construction.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index d12162c099379..c171812d8771b 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -14,25 +14,29 @@ from pandas import ( DatetimeIndex, Index, Timestamp, date_range, datetime, offsets, to_datetime) -from pandas.core.arrays import period_array +from pandas.core.arrays import ( + DatetimeArrayMixin as DatetimeArray, period_array) import pandas.util.testing as tm class TestDatetimeIndex(object): - def test_freq_validation_with_nat(self): + @pytest.mark.parametrize('dt_cls', [DatetimeIndex, DatetimeArray]) + def test_freq_validation_with_nat(self, dt_cls): # GH#11587 make sure we get a useful error message when generate_range # raises msg = ("Inferred frequency None from passed values does not conform " "to passed frequency D") with pytest.raises(ValueError, match=msg): - DatetimeIndex([pd.NaT, pd.Timestamp('2011-01-01')], freq='D') + dt_cls([pd.NaT, pd.Timestamp('2011-01-01')], freq='D') with pytest.raises(ValueError, match=msg): - DatetimeIndex([pd.NaT, pd.Timestamp('2011-01-01').value], - freq='D') + dt_cls([pd.NaT, pd.Timestamp('2011-01-01').value], + freq='D') def test_categorical_preserves_tz(self): # GH#18664 retain tz when going DTI-->Categorical-->DTI + # TODO: parametrize over DatetimeIndex/DatetimeArray + # once CategoricalIndex(DTA) works dti = pd.DatetimeIndex( [pd.NaT, '2015-01-01', '1999-04-06 15:14:13', '2015-01-01'], From 12e0f4ee38c80e0f1d1b33a92976f1ce9fd196b1 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 16:08:57 -0800 Subject: [PATCH 11/26] comment --- pandas/core/arrays/datetimes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index ae30e473c9279..e7c80ae4f43cf 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1650,6 +1650,8 @@ def maybe_convert_dtype(data, copy): "Use `data.to_timestamp()` instead") elif is_categorical_dtype(data): + # TODO: cases where we need to do another pass through this func, + # e.g. the categories are timedelta64s if isna(data).any(): # slow-path data = list(data) From 28bf2de2a0cbe15d17e6c98b617e327657236aec Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 16:09:53 -0800 Subject: [PATCH 12/26] update GH reference --- doc/source/whatsnew/v0.24.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 59995decd9873..9f8ce30ec3488 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1137,7 +1137,7 @@ Deprecations - :func:`pandas.types.is_datetimetz` is deprecated in favor of `pandas.types.is_datetime64tz` (:issue:`23917`) - Creating a :class:`TimedeltaIndex` or :class:`DatetimeIndex` by passing range arguments `start`, `end`, and `periods` is deprecated in favor of :func:`timedelta_range` and :func:`date_range` (:issue:`23919`) - Passing a string alias like ``'datetime64[ns, UTC]'`` as the `unit` parameter to :class:`DatetimeTZDtype` is deprecated. Use :class:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`). -- Passing a `time_rule` to `pandas.tseries.offsets.generate_range` is deprecated and will raise a ``TypeError`` in a future version. Pass an ``offset`` instead (:issue:`????`) +- Passing a `time_rule` to `pandas.tseries.offsets.generate_range` is deprecated and will raise a ``TypeError`` in a future version. Pass an ``offset`` instead (:issue:`24157`) .. _whatsnew_0240.deprecations.datetimelike_int_ops: From 06d0a8e9bd6011caa729448a43bd0eca785cb9a6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 17:45:24 -0800 Subject: [PATCH 13/26] make TDI catch nanos valuerror --- pandas/core/arrays/datetimelike.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index b43e449697708..92b93b6514f99 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -551,6 +551,11 @@ def _validate_frequency(cls, index, freq, **kwargs): if index.size == 0 or inferred == freq.freqstr: return None + if is_timedelta64_dtype(cls): + # if a non-fixed frequency is passed, accessing `freq.nanos` + # will raise ValueError + freq.nanos + try: on_freq = cls._generate_range(start=index[0], end=None, periods=len(index), freq=freq, From 81c7d0fd9b8f0323dac65071a6119d983fda0f61 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 7 Dec 2018 18:42:18 -0800 Subject: [PATCH 14/26] fix error message, hopefully --- pandas/core/arrays/datetimelike.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 92b93b6514f99..149929bfb695c 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -551,17 +551,16 @@ def _validate_frequency(cls, index, freq, **kwargs): if index.size == 0 or inferred == freq.freqstr: return None - if is_timedelta64_dtype(cls): - # if a non-fixed frequency is passed, accessing `freq.nanos` - # will raise ValueError - freq.nanos - try: on_freq = cls._generate_range(start=index[0], end=None, periods=len(index), freq=freq, **kwargs) assert np.array_equal(index.asi8, on_freq.asi8) - except (ValueError, AssertionError): + except (ValueError, AssertionError) as e: + if "non-fixed" in str(e): + # non-fixed frequencies are not meaningful for timedelta64; + # we retain that error message + raise e # GH#11587 if index[0] is NaT _generate_range will raise ValueError raise ValueError('Inferred frequency {infer} from passed values ' 'does not conform to passed frequency {passed}' From af303c7d1cd54ecff78fcf497d9b697b7a4ea846 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 9 Dec 2018 08:01:37 -0800 Subject: [PATCH 15/26] improve comments --- pandas/core/arrays/datetimelike.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 767537be230fd..c29dbfa4afd90 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -328,7 +328,7 @@ def __getitem__(self, key): "arrays are valid indices") if key is Ellipsis: - # GH#21282 + # GH#21282 avoid losing `freq` attribute return self.copy() getitem = self._data.__getitem__ @@ -561,7 +561,8 @@ def _validate_frequency(cls, index, freq, **kwargs): # non-fixed frequencies are not meaningful for timedelta64; # we retain that error message raise e - # GH#11587 if index[0] is NaT _generate_range will raise ValueError + # GH#11587 if index[0] is NaT _generate_range will raise + # ValueError, which we re-raise with a targeted error message raise ValueError('Inferred frequency {infer} from passed values ' 'does not conform to passed frequency {passed}' .format(infer=inferred, passed=freq.freqstr)) From 8ece68600ab6730b5938cf614150298e2637c24b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 9 Dec 2018 08:06:51 -0800 Subject: [PATCH 16/26] use take --- pandas/core/arrays/datetimes.py | 12 +++--------- pandas/tests/indexes/datetimes/test_construction.py | 10 ---------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index e7c80ae4f43cf..91a19ab8aeb48 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1650,17 +1650,11 @@ def maybe_convert_dtype(data, copy): "Use `data.to_timestamp()` instead") elif is_categorical_dtype(data): + # GH#18664 preserve tz in going DTI->Categorical->DTI # TODO: cases where we need to do another pass through this func, # e.g. the categories are timedelta64s - if isna(data).any(): - # slow-path - data = list(data) - data = np.array(data, dtype=np.object_) - copy = False - else: - # TODO: does this always make a copy? If so, set copy=False - # GH#18664 preserve tz in going DTI->Categorical->DTI - data = data.categories[data.codes] + data = data.categories.take(data.codes, fill_value=NaT) + # TODO: does this always make a copy? If so, set copy=False elif is_extension_type(data) and not is_datetime64tz_dtype(data): # Includes categorical diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index c171812d8771b..30f65c5096dea 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -49,16 +49,6 @@ def test_categorical_preserves_tz(self): result = pd.DatetimeIndex(obj) tm.assert_index_equal(result, dti) - # no-NaT case has a fastpath - dti2 = dti[1:] - ci2 = pd.CategoricalIndex(dti2) - carr2 = pd.Categorical(dti2) - cser2 = pd.Series(ci2) - - for obj in [ci2, carr2, cser2]: - result = pd.DatetimeIndex(obj) - tm.assert_index_equal(result, dti2) - def test_dti_with_period_data_raises(self): # GH#23675 data = pd.PeriodIndex(['2016Q1', '2016Q2'], freq='Q') From 30f01a032cc89e1f48654a7c3cea00b374f2b729 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 9 Dec 2018 08:07:38 -0800 Subject: [PATCH 17/26] whitespace --- pandas/tests/indexes/datetimes/test_construction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 30f65c5096dea..88c322ff7c9ff 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -41,6 +41,7 @@ def test_categorical_preserves_tz(self): dti = pd.DatetimeIndex( [pd.NaT, '2015-01-01', '1999-04-06 15:14:13', '2015-01-01'], tz='US/Eastern') + ci = pd.CategoricalIndex(dti) carr = pd.Categorical(dti) cser = pd.Series(ci) From df05c88dd02ae5be7adf909275b459a8ecd9e42d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 12 Dec 2018 20:37:05 -0800 Subject: [PATCH 18/26] do ellipsis check later --- pandas/core/arrays/datetimelike.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index c29dbfa4afd90..241f9d9de21d5 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -327,10 +327,6 @@ def __getitem__(self, key): "numpy.newaxis (`None`) and integer or boolean " "arrays are valid indices") - if key is Ellipsis: - # GH#21282 avoid losing `freq` attribute - return self.copy() - getitem = self._data.__getitem__ if is_int: val = getitem(key) @@ -355,6 +351,9 @@ def __getitem__(self, key): freq = key.step * self.freq else: freq = self.freq + elif key is Ellipsis: + # GH#21282 avoid losing `freq` attribute + freq = self.freq attribs['freq'] = freq From 8f39b23d3fad87c84a44f855dc65c2a17ddb2538 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 12 Dec 2018 20:38:30 -0800 Subject: [PATCH 19/26] Avoid AssertionError --- pandas/core/arrays/datetimelike.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 241f9d9de21d5..ed580d1287e2d 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -554,8 +554,9 @@ def _validate_frequency(cls, index, freq, **kwargs): on_freq = cls._generate_range(start=index[0], end=None, periods=len(index), freq=freq, **kwargs) - assert np.array_equal(index.asi8, on_freq.asi8) - except (ValueError, AssertionError) as e: + if not np.array_equal(index.asi8, on_freq.asi8): + raise ValueError + except ValueError as e: if "non-fixed" in str(e): # non-fixed frequencies are not meaningful for timedelta64; # we retain that error message From e958ce606c2486125c2df27fd9bc97af94c04de7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 12 Dec 2018 20:39:11 -0800 Subject: [PATCH 20/26] remove resolved comment --- pandas/core/arrays/datetimes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 1bf9d04b382d2..2ecbf9f0ff847 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1659,7 +1659,7 @@ def maybe_convert_dtype(data, copy): # TODO: cases where we need to do another pass through this func, # e.g. the categories are timedelta64s data = data.categories.take(data.codes, fill_value=NaT) - # TODO: does this always make a copy? If so, set copy=False + copy = False elif is_extension_type(data) and not is_datetime64tz_dtype(data): # Includes categorical From 36d37e488d6d669d037b4e87cd39985bee920918 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 13 Dec 2018 09:57:57 -0800 Subject: [PATCH 21/26] more whatsnew --- doc/source/whatsnew/v0.24.0.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index accc0eae148df..66526451e7b84 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1310,6 +1310,8 @@ Datetimelike - Bug in :class:`Index` where calling ``np.array(dtindex, dtype=object)`` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) - Bug in :class:`Categorical.__setitem__` not allowing setting with another ``Categorical`` when both are undordered and have the same categories, but in a different order (:issue:`24142`) - Bug in :func:`date_range` where using dates with millisecond resolution or higher could return incorrect values or the wrong number of values in the index (:issue:`24110`) +- Bug in :class:`DatetimeIndex` where constructing a :class:`DatetimeIndex` from a :class:`Categorical` or :class:`CategoricalIndex` would incorrectly drop timezone information (:issue:`18664`) +- Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would incorrectly lose the index's ``freq`` attribute (:issue:`21282`) Timedelta ^^^^^^^^^ From 3fc1c19e7f3691990f9e6e7a382ceeeb3fc4b859 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 13 Dec 2018 09:59:22 -0800 Subject: [PATCH 22/26] comment harder --- pandas/core/arrays/datetimelike.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index ed580d1287e2d..7177b6f786aa6 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -352,7 +352,8 @@ def __getitem__(self, key): else: freq = self.freq elif key is Ellipsis: - # GH#21282 avoid losing `freq` attribute + # GH#21282 indexing with Ellipsis is similar to a full slice, + # should preserve `freq` attribute freq = self.freq attribs['freq'] = freq From 88f7094bdf17751eee5f153fd03966593348a2cf Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 13 Dec 2018 10:04:33 -0800 Subject: [PATCH 23/26] comment harder --- pandas/core/arrays/datetimelike.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 7177b6f786aa6..a6eacc3bb4bfd 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -562,8 +562,11 @@ def _validate_frequency(cls, index, freq, **kwargs): # non-fixed frequencies are not meaningful for timedelta64; # we retain that error message raise e - # GH#11587 if index[0] is NaT _generate_range will raise - # ValueError, which we re-raise with a targeted error message + # GH#11587 the main way this is reached is if the `np.array_equal` + # check above is False. This can also be reached if index[0] + # is `NaT`, in which case the call to `cls._generate_range` will + # raise a ValueError, which we re-raise with a more targeted + # message. raise ValueError('Inferred frequency {infer} from passed values ' 'does not conform to passed frequency {passed}' .format(infer=inferred, passed=freq.freqstr)) From e178bb9707d8e20a1f37f11629a3d5e8a003dc3b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 13 Dec 2018 10:54:18 -0800 Subject: [PATCH 24/26] remove time_rule entirely --- doc/source/whatsnew/v0.24.0.rst | 2 +- .../tests/indexes/datetimes/test_date_range.py | 6 ++---- pandas/tests/tseries/offsets/test_offsets.py | 10 ---------- pandas/tseries/offsets.py | 16 +++------------- 4 files changed, 6 insertions(+), 28 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 66526451e7b84..2a17e7baa0b19 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -380,6 +380,7 @@ Backwards incompatible API changes - ``max_rows`` and ``max_cols`` parameters removed from :class:`HTMLFormatter` since truncation is handled by :class:`DataFrameFormatter` (:issue:`23818`) - :meth:`read_csv` will now raise a ``ValueError`` if a column with missing values is declared as having dtype ``bool`` (:issue:`20591`) - The column order of the resultant :class:`DataFrame` from :meth:`MultiIndex.to_frame` is now guaranteed to match the :attr:`MultiIndex.names` order. (:issue:`22420`) +- :func:`pd.offsets.generate_range` argument ``time_rule`` has been removed; use ``offset`` instead (:issue:`24157`) Percentage change on groupby changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1132,7 +1133,6 @@ Deprecations - Passing a string alias like ``'datetime64[ns, UTC]'`` as the `unit` parameter to :class:`DatetimeTZDtype` is deprecated. Use :class:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`). - In :meth:`Series.where` with Categorical data, providing an ``other`` that is not present in the categories is deprecated. Convert the categorical to a different dtype or add the ``other`` to the categories first (:issue:`24077`). - :meth:`Series.clip_lower`, :meth:`Series.clip_upper`, :meth:`DataFrame.clip_lower` and :meth:`DataFrame.clip_upper` are deprecated and will be removed in a future version. Use ``Series.clip(lower=threshold)``, ``Series.clip(upper=threshold)`` and the equivalent ``DataFrame`` methods (:issue:`24203`) -- Passing a `time_rule` to `pandas.tseries.offsets.generate_range` is deprecated and will raise a ``TypeError`` in a future version. Pass an ``offset`` instead (:issue:`24157`) .. _whatsnew_0240.deprecations.datetimelike_int_ops: diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index 044e112285359..a39100b3ec204 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -541,14 +541,12 @@ class TestGenRangeGeneration(object): def test_generate(self): rng1 = list(generate_range(START, END, offset=BDay())) - with tm.assert_produces_warning(FutureWarning): - rng2 = list(generate_range(START, END, time_rule='B')) + rng2 = list(generate_range(START, END, offset='B')) assert rng1 == rng2 def test_generate_cday(self): rng1 = list(generate_range(START, END, offset=CDay())) - with tm.assert_produces_warning(FutureWarning): - rng2 = list(generate_range(START, END, time_rule='C')) + rng2 = list(generate_range(START, END, offset='C')) assert rng1 == rng2 def test_1(self): diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 9ac21d3587dc9..456e0b10e5a96 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -49,16 +49,6 @@ class WeekDay(object): #### -def test_time_rule_deprecated(): - start = datetime(2007, 10, 1) - end = datetime(2012, 4, 9) - - with tm.assert_produces_warning(FutureWarning): - # Note: generate_range returns a generator, and the warning is not - # issued until the first __next__ call - list(offsets.generate_range(start=start, end=end, time_rule="W")) - - def test_to_m8(): valb = datetime(2007, 10, 1) valu = _to_m8(valb) diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 107b9e8f186d8..d3b21be86e701 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2458,8 +2458,7 @@ class Nano(Tick): # --------------------------------------------------------------------- -def generate_range(start=None, end=None, periods=None, - offset=BDay(), time_rule=None): +def generate_range(start=None, end=None, periods=None, offset=BDay()): """ Generates a sequence of dates corresponding to the specified time offset. Similar to dateutil.rrule except uses pandas DateOffset @@ -2471,8 +2470,6 @@ def generate_range(start=None, end=None, periods=None, end : datetime (default None) periods : int, (default None) offset : DateOffset, (default BDay()) - time_rule : (legacy) name of DateOffset object to be used, optional - Corresponds with names expected by tseries.frequencies.get_offset Notes ----- @@ -2480,20 +2477,13 @@ def generate_range(start=None, end=None, periods=None, * At least two of (start, end, periods) must be specified. * If both start and end are specified, the returned dates will satisfy start <= date <= end. - * If both time_rule and offset are specified, time_rule supersedes offset. Returns ------- dates : generator object - """ - if time_rule is not None: - warnings.warn("`time_rule` is deprecated and will be removed in a " - "future version. Use `offset` instead.", - FutureWarning, stacklevel=2) - from pandas.tseries.frequencies import get_offset - - offset = get_offset(time_rule) + from pandas.tseries.frequencies import to_offset + offset = to_offset(offset) start = to_datetime(start) end = to_datetime(end) From 50f6b7e5750e39c1180b5f207ef187489f8b9fa2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 13 Dec 2018 11:04:39 -0800 Subject: [PATCH 25/26] whatsnew --- doc/source/whatsnew/v0.24.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 2a17e7baa0b19..b04d2eeba1ed0 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1312,6 +1312,7 @@ Datetimelike - Bug in :func:`date_range` where using dates with millisecond resolution or higher could return incorrect values or the wrong number of values in the index (:issue:`24110`) - Bug in :class:`DatetimeIndex` where constructing a :class:`DatetimeIndex` from a :class:`Categorical` or :class:`CategoricalIndex` would incorrectly drop timezone information (:issue:`18664`) - Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would incorrectly lose the index's ``freq`` attribute (:issue:`21282`) +- Clarified error message produced when passing an incorrect ``freq`` argument to :class:`DatetimeIndex` with ``NaT`` as the first entry in the passed data (:issue:`11587`) Timedelta ^^^^^^^^^ From b927925a14def3d3d8710493517d34ebe950c040 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 13 Dec 2018 11:50:36 -0800 Subject: [PATCH 26/26] flake8 fixup --- pandas/tseries/offsets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index d3b21be86e701..cff9556a4230e 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2,7 +2,6 @@ from datetime import date, datetime, timedelta import functools import operator -import warnings from dateutil.easter import easter import numpy as np