From 67ab32f3d858a826f07a7e3cd8114a6a42a65aaf Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 16:26:51 -0700 Subject: [PATCH 01/28] TST: fix xfailed arithmetic tests --- pandas/core/ops/dispatch.py | 8 ++++-- pandas/tests/arithmetic/test_timedelta64.py | 32 +++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pandas/core/ops/dispatch.py b/pandas/core/ops/dispatch.py index 637f0fa1d52e9..a7dcdd4f9d585 100644 --- a/pandas/core/ops/dispatch.py +++ b/pandas/core/ops/dispatch.py @@ -65,8 +65,12 @@ def should_series_dispatch(left, right, op): ldtype = left.dtypes.iloc[0] rdtype = right.dtypes.iloc[0] - if (is_timedelta64_dtype(ldtype) and is_integer_dtype(rdtype)) or ( - is_timedelta64_dtype(rdtype) and is_integer_dtype(ldtype) + if ( + is_timedelta64_dtype(ldtype) + and (is_integer_dtype(rdtype) or is_object_dtype(rdtype)) + ) or ( + is_timedelta64_dtype(rdtype) + and (is_integer_dtype(ldtype) or is_object_dtype(ldtype)) ): # numpy integer dtypes as timedelta64 dtypes in this scenario return True diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index b64a52a772419..af0db56204729 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -541,16 +541,17 @@ def test_tda_add_sub_index(self): expected = tdi - tdi tm.assert_index_equal(result, expected) - def test_tda_add_dt64_object_array(self, box_df_fail, tz_naive_fixture): + def test_tda_add_dt64_object_array(self, box_with_array, tz_naive_fixture): # Result should be cast back to DatetimeArray + box = box_with_array dti = pd.date_range("2016-01-01", periods=3, tz=tz_naive_fixture) dti = dti._with_freq(None) tdi = dti - dti - obj = tm.box_expected(tdi, box_df_fail) - other = tm.box_expected(dti, box_df_fail) + obj = tm.box_expected(tdi, box) + other = tm.box_expected(dti, box) - warn = PerformanceWarning if box_df_fail is not pd.DataFrame else None + warn = PerformanceWarning if box is not pd.DataFrame else None with tm.assert_produces_warning(warn): result = obj + other.astype(object) tm.assert_equal(result, other) @@ -1395,33 +1396,40 @@ def test_td64arr_sub_offset_array(self, box_with_array): @pytest.mark.parametrize( "names", [(None, None, None), ("foo", "bar", None), ("foo", "foo", "foo")] ) - def test_td64arr_with_offset_series(self, names, box_df_fail): + def test_td64arr_with_offset_series(self, names, box_with_array): # GH#18849 - box = box_df_fail + box = box_with_array box2 = Series if box in [pd.Index, tm.to_array] else box - exname = names[2] if box is not tm.to_array else names[1] + + if box is pd.DataFrame: + # Since we are operating with a DataFrame and a non-DataFrame, + # the non-DataFrame is cast to Series and its name ignored. + exname = names[0] + elif box is tm.to_array: + exname = names[1] + else: + exname = names[2] tdi = TimedeltaIndex(["1 days 00:00:00", "3 days 04:00:00"], name=names[0]) other = Series([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)], name=names[1]) expected_add = Series([tdi[n] + other[n] for n in range(len(tdi))], name=exname) - tdi = tm.box_expected(tdi, box) + obj = tm.box_expected(tdi, box) expected_add = tm.box_expected(expected_add, box2) with tm.assert_produces_warning(PerformanceWarning): - res = tdi + other + res = obj + other tm.assert_equal(res, expected_add) with tm.assert_produces_warning(PerformanceWarning): - res2 = other + tdi + res2 = other + obj tm.assert_equal(res2, expected_add) - # TODO: separate/parametrize add/sub test? expected_sub = Series([tdi[n] - other[n] for n in range(len(tdi))], name=exname) expected_sub = tm.box_expected(expected_sub, box2) with tm.assert_produces_warning(PerformanceWarning): - res3 = tdi - other + res3 = obj - other tm.assert_equal(res3, expected_sub) @pytest.mark.parametrize("obox", [np.array, pd.Index, pd.Series]) From ae3837c7831ef9ba226c427a6bf92c33fc147da7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 12:37:44 -0700 Subject: [PATCH 02/28] TST: strictify xfails --- pandas/tests/arithmetic/test_timedelta64.py | 18 +++++++++--------- pandas/tests/extension/base/ops.py | 10 ++++++++-- pandas/tests/extension/test_boolean.py | 12 +++++++++++- pandas/tests/extension/test_categorical.py | 8 ++++++++ pandas/tests/extension/test_datetime.py | 9 +++++++++ pandas/tests/extension/test_period.py | 9 +++++++++ pandas/tests/frame/test_cumulative.py | 8 ++------ pandas/tests/series/test_cumulative.py | 8 ++------ pandas/tests/test_expressions.py | 2 -- 9 files changed, 58 insertions(+), 26 deletions(-) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index af0db56204729..b97c25593d850 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -1196,9 +1196,11 @@ def test_td64arr_sub_td64_array(self, box_with_array): result = tdarr - tdi tm.assert_equal(result, expected) - def test_td64arr_add_sub_tdi(self, box, names): + def test_td64arr_add_sub_tdi(self, box_with_array, names): # GH#17250 make sure result dtype is correct # GH#19043 make sure names are propagated correctly + box = box_with_array + if box is pd.DataFrame and names[1] != names[0]: pytest.skip( "Name propagation for DataFrame does not behave like " @@ -1206,6 +1208,7 @@ def test_td64arr_add_sub_tdi(self, box, names): ) tdi = TimedeltaIndex(["0 days", "1 day"], name=names[0]) + tdi = np.array(tdi) if box is tm.to_array else tdi ser = Series([Timedelta(hours=3), Timedelta(hours=4)], name=names[1]) expected = Series( [Timedelta(hours=3), Timedelta(days=1, hours=4)], name=names[2] @@ -1300,8 +1303,10 @@ def test_td64arr_sub_timedeltalike(self, two_hours, box_with_array): # ------------------------------------------------------------------ # __add__/__sub__ with DateOffsets and arrays of DateOffsets - def test_td64arr_add_offset_index(self, names, box): + def test_td64arr_add_offset_index(self, names, box_with_array): # GH#18849, GH#19744 + box = box_with_array + if box is pd.DataFrame and names[1] != names[0]: pytest.skip( "Name propagation for DataFrame does not behave like " @@ -1310,6 +1315,7 @@ def test_td64arr_add_offset_index(self, names, box): tdi = TimedeltaIndex(["1 days 00:00:00", "3 days 04:00:00"], name=names[0]) other = pd.Index([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)], name=names[1]) + other = np.array(other) if box is tm.to_array else other expected = TimedeltaIndex( [tdi[n] + other[n] for n in range(len(tdi))], freq="infer", name=names[2] @@ -1348,16 +1354,13 @@ def test_td64arr_add_offset_array(self, box_with_array): res2 = other + tdi tm.assert_equal(res2, expected) - @pytest.mark.parametrize( - "names", [(None, None, None), ("foo", "bar", None), ("foo", "foo", "foo")] - ) def test_td64arr_sub_offset_index(self, names, box_with_array): # GH#18824, GH#19744 box = box_with_array xbox = box if box is not tm.to_array else pd.Index exname = names[2] if box is not tm.to_array else names[1] - if box is pd.DataFrame and names[1] == "bar": + if box is pd.DataFrame and names[1] != names[0]: pytest.skip( "Name propagation for DataFrame does not behave like " "it does for Index/Series" @@ -1393,9 +1396,6 @@ def test_td64arr_sub_offset_array(self, box_with_array): res = tdi - other tm.assert_equal(res, expected) - @pytest.mark.parametrize( - "names", [(None, None, None), ("foo", "bar", None), ("foo", "foo", "foo")] - ) def test_td64arr_with_offset_series(self, names, box_with_array): # GH#18849 box = box_with_array diff --git a/pandas/tests/extension/base/ops.py b/pandas/tests/extension/base/ops.py index 4009041218ac2..d3b6472044ea5 100644 --- a/pandas/tests/extension/base/ops.py +++ b/pandas/tests/extension/base/ops.py @@ -29,8 +29,14 @@ def check_opname(self, s, op_name, other, exc=Exception): def _check_op(self, s, op, other, op_name, exc=NotImplementedError): if exc is None: result = op(s, other) - expected = s.combine(other, op) - self.assert_series_equal(result, expected) + if isinstance(s, pd.DataFrame): + if len(s.columns) != 1: + raise NotImplementedError + expected = s.iloc[:, 0].combine(other, op).to_frame() + self.assert_frame_equal(result, expected) + else: + expected = s.combine(other, op) + self.assert_series_equal(result, expected) else: with pytest.raises(exc): op(s, other) diff --git a/pandas/tests/extension/test_boolean.py b/pandas/tests/extension/test_boolean.py index e2331b69916fb..303764bff50b6 100644 --- a/pandas/tests/extension/test_boolean.py +++ b/pandas/tests/extension/test_boolean.py @@ -102,13 +102,15 @@ class TestMissing(base.BaseMissingTests): class TestArithmeticOps(base.BaseArithmeticOpsTests): + implements = {"__sub__", "__rsub__"} + def check_opname(self, s, op_name, other, exc=None): # overwriting to indicate ops don't raise an error super().check_opname(s, op_name, other, exc=None) def _check_op(self, s, op, other, op_name, exc=NotImplementedError): if exc is None: - if op_name in ("__sub__", "__rsub__"): + if op_name in self.implements: # subtraction for bools raises TypeError (but not yet in 1.13) if _np_version_under1p14: pytest.skip("__sub__ does not yet raise in numpy 1.13") @@ -151,6 +153,14 @@ def test_error(self, data, all_arithmetic_operators): # other specific errors tested in the boolean array specific tests pass + def test_arith_frame_with_scalar(self, data, all_arithmetic_operators): + # frame & scalar + op_name = all_arithmetic_operators + if op_name in self.implements: + super().test_arith_frame_with_scalar(data, all_arithmetic_operators) + else: + pytest.xfail("_reduce needs implementation") + class TestComparisonOps(base.BaseComparisonOpsTests): def check_opname(self, s, op_name, other, exc=None): diff --git a/pandas/tests/extension/test_categorical.py b/pandas/tests/extension/test_categorical.py index 059d3453995bd..91d7cea90e0b3 100644 --- a/pandas/tests/extension/test_categorical.py +++ b/pandas/tests/extension/test_categorical.py @@ -242,6 +242,14 @@ def test_consistent_casting(self, dtype, expected): class TestArithmeticOps(base.BaseArithmeticOpsTests): + def test_arith_frame_with_scalar(self, data, all_arithmetic_operators): + # frame & scalar + op_name = all_arithmetic_operators + if op_name != "__rmod__": + super().test_arith_frame_with_scalar(data, all_arithmetic_operators) + else: + pytest.skip("rmod never called when string is first argument") + def test_arith_series_with_scalar(self, data, all_arithmetic_operators): op_name = all_arithmetic_operators diff --git a/pandas/tests/extension/test_datetime.py b/pandas/tests/extension/test_datetime.py index 3aa188098620d..e026809f7e611 100644 --- a/pandas/tests/extension/test_datetime.py +++ b/pandas/tests/extension/test_datetime.py @@ -111,6 +111,15 @@ def test_array_interface(self, data): class TestArithmeticOps(BaseDatetimeTests, base.BaseArithmeticOpsTests): implements = {"__sub__", "__rsub__"} + def test_arith_frame_with_scalar(self, data, all_arithmetic_operators): + # frame & scalar + if all_arithmetic_operators in self.implements: + df = pd.DataFrame({"A": data}) + self.check_opname(df, all_arithmetic_operators, data[0], exc=None) + else: + # ... but not the rest. + super().test_arith_frame_with_scalar(data, all_arithmetic_operators) + def test_arith_series_with_scalar(self, data, all_arithmetic_operators): if all_arithmetic_operators in self.implements: s = pd.Series(data) diff --git a/pandas/tests/extension/test_period.py b/pandas/tests/extension/test_period.py index c439b8b5ed319..11b41eedd5d51 100644 --- a/pandas/tests/extension/test_period.py +++ b/pandas/tests/extension/test_period.py @@ -84,6 +84,15 @@ class TestInterface(BasePeriodTests, base.BaseInterfaceTests): class TestArithmeticOps(BasePeriodTests, base.BaseArithmeticOpsTests): implements = {"__sub__", "__rsub__"} + def test_arith_frame_with_scalar(self, data, all_arithmetic_operators): + # frame & scalar + if all_arithmetic_operators in self.implements: + df = pd.DataFrame({"A": data}) + self.check_opname(df, all_arithmetic_operators, data[0], exc=None) + else: + # ... but not the rest. + super().test_arith_frame_with_scalar(data, all_arithmetic_operators) + def test_arith_series_with_scalar(self, data, all_arithmetic_operators): # we implement substitution... if all_arithmetic_operators in self.implements: diff --git a/pandas/tests/frame/test_cumulative.py b/pandas/tests/frame/test_cumulative.py index 1b7e70dd28c63..5441505a920db 100644 --- a/pandas/tests/frame/test_cumulative.py +++ b/pandas/tests/frame/test_cumulative.py @@ -75,9 +75,7 @@ def test_cumprod(self, datetime_frame): df.cumprod(1) @pytest.mark.xfail( - _is_numpy_dev, - reason="https://github.com/pandas-dev/pandas/issues/31992", - strict=False, + _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummin(self, datetime_frame): datetime_frame.iloc[5:10, 0] = np.nan @@ -103,9 +101,7 @@ def test_cummin(self, datetime_frame): assert np.shape(cummin_xs) == np.shape(datetime_frame) @pytest.mark.xfail( - _is_numpy_dev, - reason="https://github.com/pandas-dev/pandas/issues/31992", - strict=False, + _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummax(self, datetime_frame): datetime_frame.iloc[5:10, 0] = np.nan diff --git a/pandas/tests/series/test_cumulative.py b/pandas/tests/series/test_cumulative.py index 259c5d53c5492..e1a1f093f08b7 100644 --- a/pandas/tests/series/test_cumulative.py +++ b/pandas/tests/series/test_cumulative.py @@ -39,9 +39,7 @@ def test_cumprod(self, datetime_series): _check_accum_op("cumprod", datetime_series) @pytest.mark.xfail( - _is_numpy_dev, - reason="https://github.com/pandas-dev/pandas/issues/31992", - strict=False, + _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummin(self, datetime_series): tm.assert_numpy_array_equal( @@ -57,9 +55,7 @@ def test_cummin(self, datetime_series): tm.assert_series_equal(result, expected) @pytest.mark.xfail( - _is_numpy_dev, - reason="https://github.com/pandas-dev/pandas/issues/31992", - strict=False, + _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummax(self, datetime_series): tm.assert_numpy_array_equal( diff --git a/pandas/tests/test_expressions.py b/pandas/tests/test_expressions.py index fadab5d821470..b0a369ea65c94 100644 --- a/pandas/tests/test_expressions.py +++ b/pandas/tests/test_expressions.py @@ -363,8 +363,6 @@ def test_bool_ops_column_name_dtype(self, test_input, expected): @pytest.mark.parametrize("axis", (0, 1)) def test_frame_series_axis(self, axis, arith): # GH#26736 Dataframe.floordiv(Series, axis=1) fails - if axis == 1 and arith == "floordiv": - pytest.xfail("'floordiv' does not succeed with axis=1 #27636") df = self.frame if axis == 1: From 63cbaa9866fd72d6e9732b1f6c72b9b3f3573e8d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 12:40:23 -0700 Subject: [PATCH 03/28] strictify --- pandas/tests/scalar/timedelta/test_arithmetic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/scalar/timedelta/test_arithmetic.py b/pandas/tests/scalar/timedelta/test_arithmetic.py index eb22b715f9f4d..1663fa65c7dfe 100644 --- a/pandas/tests/scalar/timedelta/test_arithmetic.py +++ b/pandas/tests/scalar/timedelta/test_arithmetic.py @@ -419,7 +419,6 @@ def test_td_div_numeric_scalar(self): _is_numpy_dev, raises=RuntimeWarning, reason="https://github.com/pandas-dev/pandas/issues/31992", - strict=False, ), ), float("nan"), From 5338c37049305221a284cd84f3931fd2b905b901 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 12:43:02 -0700 Subject: [PATCH 04/28] strictify xfail --- pandas/tests/io/parser/test_usecols.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/parser/test_usecols.py b/pandas/tests/io/parser/test_usecols.py index 979eb4702cc84..a9ed4370ddc9f 100644 --- a/pandas/tests/io/parser/test_usecols.py +++ b/pandas/tests/io/parser/test_usecols.py @@ -558,11 +558,12 @@ def test_raises_on_usecols_names_mismatch(all_parsers, usecols, kwargs, expected tm.assert_frame_equal(result, expected) -@pytest.mark.xfail( - reason="see gh-16469: works on the C engine but not the Python engine", strict=False -) @pytest.mark.parametrize("usecols", [["A", "C"], [0, 2]]) -def test_usecols_subset_names_mismatch_orig_columns(all_parsers, usecols): +def test_usecols_subset_names_mismatch_orig_columns(all_parsers, usecols, request): + if all_parsers.engine != "c": + reason = "see gh-16469: works on the C engine but not the Python engine" + request.node.add_marker(pytest.mark.xfail(reason=reason)) + data = "a,b,c,d\n1,2,3,4\n5,6,7,8" names = ["A", "B", "C", "D"] parser = all_parsers From 5ba2112b7e84a9d956833ac4476949331145b8b3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 12:44:06 -0700 Subject: [PATCH 05/28] stricter --- pandas/tests/io/parser/test_usecols.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/parser/test_usecols.py b/pandas/tests/io/parser/test_usecols.py index a9ed4370ddc9f..d4e049cc3fcc2 100644 --- a/pandas/tests/io/parser/test_usecols.py +++ b/pandas/tests/io/parser/test_usecols.py @@ -562,7 +562,8 @@ def test_raises_on_usecols_names_mismatch(all_parsers, usecols, kwargs, expected def test_usecols_subset_names_mismatch_orig_columns(all_parsers, usecols, request): if all_parsers.engine != "c": reason = "see gh-16469: works on the C engine but not the Python engine" - request.node.add_marker(pytest.mark.xfail(reason=reason)) + # Number of passed names did not match number of header fields in the file + request.node.add_marker(pytest.mark.xfail(reason=reason, raises=ValueError)) data = "a,b,c,d\n1,2,3,4\n5,6,7,8" names = ["A", "B", "C", "D"] From 13d93bff4454f2725fd8ef3c18e938aeaf34dccb Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 13:43:19 -0700 Subject: [PATCH 06/28] fix xfail --- pandas/tests/groupby/test_categorical.py | 3 ++- pandas/tests/test_downstream.py | 7 +++---- pandas/tests/tseries/offsets/test_offsets_properties.py | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index 8e4a7141875bb..ee0af141d0f95 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -13,6 +13,7 @@ Index, MultiIndex, Series, + _np_version_under1p17, qcut, ) import pandas._testing as tm @@ -210,7 +211,7 @@ def test_level_get_group(observed): # GH#21636 flaky on py37; may be related to older numpy, see discussion # https://github.com/MacPython/pandas-wheels/pull/64 -@pytest.mark.xfail(PY37, reason="Flaky, GH-27902", strict=False) +@pytest.mark.xfail(PY37 and _np_version_under1p17, reason="Flaky, GH-27902") @pytest.mark.parametrize("ordered", [True, False]) def test_apply(ordered): # GH 10138 diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 57542aa3bc7f6..20a00162a4464 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -69,6 +69,7 @@ def test_oo_optimizable(): @tm.network # Cython import warning +@pytest.mark.filterwarnings("ignore:pandas.util.testing is deprecated") @pytest.mark.filterwarnings("ignore:can't:ImportWarning") @pytest.mark.filterwarnings( # patsy needs to update their imports @@ -140,19 +141,17 @@ def test_pyarrow(df): tm.assert_frame_equal(result, df) -@pytest.mark.xfail(reason="pandas-wheels-50", strict=False) def test_missing_required_dependency(): # GH 23868 # To ensure proper isolation, we pass these flags # -S : disable site-packages # -s : disable user site-packages # -E : disable PYTHON* env vars, especially PYTHONPATH - # And, that's apparently not enough, so we give up. # https://github.com/MacPython/pandas-wheels/pull/50 - call = ["python", "-sSE", "-c", "import pandas"] + call = [sys.executable, "-sSE", "-c", "import pandas"] msg = ( - r"Command '\['python', '-sSE', '-c', 'import pandas'\]' " + rf"Command '\['{sys.executable}', '-sSE', '-c', 'import pandas'\]' " "returned non-zero exit status 1." ) diff --git a/pandas/tests/tseries/offsets/test_offsets_properties.py b/pandas/tests/tseries/offsets/test_offsets_properties.py index 716d3ff3faf1c..025174f6f1d9a 100644 --- a/pandas/tests/tseries/offsets/test_offsets_properties.py +++ b/pandas/tests/tseries/offsets/test_offsets_properties.py @@ -14,6 +14,8 @@ from hypothesis.extra.pytz import timezones as pytz_timezones import pytest +from pandas.compat import is_platform_mac + import pandas as pd from pandas import Timestamp @@ -86,7 +88,7 @@ # Based on CI runs: Always passes on OSX, fails on Linux, sometimes on Windows -@pytest.mark.xfail(strict=False, reason="inconsistent between OSs, Pythons") +@pytest.mark.xfail(not is_platform_mac(), reason="inconsistent between OSs, Pythons") @given(gen_random_datetime, gen_yqm_offset) def test_on_offset_implementations(dt, offset): assume(not offset.normalize) From a8328bcaeb14f173cbe1afed08a65a3edc6b3be3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 14:56:52 -0700 Subject: [PATCH 07/28] tighten --- pandas/tests/tseries/offsets/test_offsets_properties.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tseries/offsets/test_offsets_properties.py b/pandas/tests/tseries/offsets/test_offsets_properties.py index 025174f6f1d9a..0dfdf2992432d 100644 --- a/pandas/tests/tseries/offsets/test_offsets_properties.py +++ b/pandas/tests/tseries/offsets/test_offsets_properties.py @@ -14,7 +14,7 @@ from hypothesis.extra.pytz import timezones as pytz_timezones import pytest -from pandas.compat import is_platform_mac +from pandas.compat import is_platform_windows import pandas as pd from pandas import Timestamp @@ -88,7 +88,7 @@ # Based on CI runs: Always passes on OSX, fails on Linux, sometimes on Windows -@pytest.mark.xfail(not is_platform_mac(), reason="inconsistent between OSs, Pythons") +@pytest.mark.xfail(is_platform_windows(), reason="inconsistent between OSs, Pythons") @given(gen_random_datetime, gen_yqm_offset) def test_on_offset_implementations(dt, offset): assume(not offset.normalize) From 7ed3c672a6be7e749b8fc4c45acbc8c5f25e9218 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 15:01:02 -0700 Subject: [PATCH 08/28] take it all back --- pandas/tests/test_downstream.py | 6 ++++-- pandas/tests/tseries/offsets/test_offsets_properties.py | 4 ---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 20a00162a4464..b671564eb77fe 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -148,10 +148,12 @@ def test_missing_required_dependency(): # -s : disable user site-packages # -E : disable PYTHON* env vars, especially PYTHONPATH # https://github.com/MacPython/pandas-wheels/pull/50 - call = [sys.executable, "-sSE", "-c", "import pandas"] + + pyexe = sys.executable.replace("\\", "/") + call = [pyexe, "-sSE", "-c", "import pandas"] msg = ( - rf"Command '\['{sys.executable}', '-sSE', '-c', 'import pandas'\]' " + rf"Command '\['{pyexe}', '-sSE', '-c', 'import pandas'\]' " "returned non-zero exit status 1." ) diff --git a/pandas/tests/tseries/offsets/test_offsets_properties.py b/pandas/tests/tseries/offsets/test_offsets_properties.py index 0dfdf2992432d..082aa45f959ff 100644 --- a/pandas/tests/tseries/offsets/test_offsets_properties.py +++ b/pandas/tests/tseries/offsets/test_offsets_properties.py @@ -14,8 +14,6 @@ from hypothesis.extra.pytz import timezones as pytz_timezones import pytest -from pandas.compat import is_platform_windows - import pandas as pd from pandas import Timestamp @@ -87,8 +85,6 @@ # Offset-specific behaviour tests -# Based on CI runs: Always passes on OSX, fails on Linux, sometimes on Windows -@pytest.mark.xfail(is_platform_windows(), reason="inconsistent between OSs, Pythons") @given(gen_random_datetime, gen_yqm_offset) def test_on_offset_implementations(dt, offset): assume(not offset.normalize) From 3f4fcbf64dd004cc15d6af9995e4eb68ac960fad Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 15:11:49 -0700 Subject: [PATCH 09/28] tighten tests --- pandas/tests/arithmetic/test_datetime64.py | 37 +++++++++++---------- pandas/tests/arithmetic/test_timedelta64.py | 14 ++++++-- pandas/tests/frame/test_cumulative.py | 8 +++-- pandas/tests/series/test_cumulative.py | 8 +++-- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 8c937064c0493..6e116f56d1494 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -529,9 +529,9 @@ def test_dti_cmp_nat_behaves_like_float_cmp_nan(self): "op", [operator.eq, operator.ne, operator.gt, operator.ge, operator.lt, operator.le], ) - def test_comparison_tzawareness_compat(self, op, box_df_fail): + def test_comparison_tzawareness_compat(self, op, box_with_array): # GH#18162 - box = box_df_fail + box = box_with_array dr = pd.date_range("2016-01-01", periods=6) dz = dr.tz_localize("US/Pacific") @@ -543,34 +543,35 @@ def test_comparison_tzawareness_compat(self, op, box_df_fail): with pytest.raises(TypeError, match=msg): op(dr, dz) - # FIXME: DataFrame case fails to raise for == and !=, wrong - # message for inequalities + if box is pd.DataFrame: + tolist = lambda x: x.astype(object).values.tolist()[0] + else: + tolist = list + with pytest.raises(TypeError, match=msg): - op(dr, list(dz)) + op(dr, tolist(dz)) with pytest.raises(TypeError, match=msg): - op(dr, np.array(list(dz), dtype=object)) + op(dr, np.array(tolist(dz), dtype=object)) with pytest.raises(TypeError, match=msg): op(dz, dr) - # FIXME: DataFrame case fails to raise for == and !=, wrong - # message for inequalities with pytest.raises(TypeError, match=msg): - op(dz, list(dr)) + op(dz, tolist(dr)) with pytest.raises(TypeError, match=msg): - op(dz, np.array(list(dr), dtype=object)) + op(dz, np.array(tolist(dr), dtype=object)) # The aware==aware and naive==naive comparisons should *not* raise assert np.all(dr == dr) - assert np.all(dr == list(dr)) - assert np.all(list(dr) == dr) - assert np.all(np.array(list(dr), dtype=object) == dr) - assert np.all(dr == np.array(list(dr), dtype=object)) + assert np.all(dr == tolist(dr)) + assert np.all(tolist(dr) == dr) + assert np.all(np.array(tolist(dr), dtype=object) == dr) + assert np.all(dr == np.array(tolist(dr), dtype=object)) assert np.all(dz == dz) - assert np.all(dz == list(dz)) - assert np.all(list(dz) == dz) - assert np.all(np.array(list(dz), dtype=object) == dz) - assert np.all(dz == np.array(list(dz), dtype=object)) + assert np.all(dz == tolist(dz)) + assert np.all(tolist(dz) == dz) + assert np.all(np.array(tolist(dz), dtype=object) == dz) + assert np.all(dz == np.array(tolist(dz), dtype=object)) @pytest.mark.parametrize( "op", diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index b97c25593d850..e3eec1f781948 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -544,6 +544,7 @@ def test_tda_add_sub_index(self): def test_tda_add_dt64_object_array(self, box_with_array, tz_naive_fixture): # Result should be cast back to DatetimeArray box = box_with_array + dti = pd.date_range("2016-01-01", periods=3, tz=tz_naive_fixture) dti = dti._with_freq(None) tdi = dti - dti @@ -2038,9 +2039,13 @@ def test_td64arr_div_numeric_array(self, box_with_array, vector, any_real_dtype) with pytest.raises(TypeError, match=pattern): vector.astype(object) / tdser - def test_td64arr_mul_int_series(self, box_df_fail, names): + def test_td64arr_mul_int_series(self, box_with_array, names, request): # GH#19042 test for correct name attachment - box = box_df_fail # broadcasts along wrong axis, but doesn't raise + box = box_with_array + if box_with_array is pd.DataFrame and names[2] is None: + reason = "broadcasts along wrong axis, but doesn't raise" + request.node.add_marker(pytest.mark.xfail(reason=reason)) + exname = names[2] if box is not tm.to_array else names[1] tdi = TimedeltaIndex( @@ -2064,7 +2069,10 @@ def test_td64arr_mul_int_series(self, box_df_fail, names): # The direct operation tdi * ser still needs to be fixed. result = ser.__rmul__(tdi) - tm.assert_equal(result, expected) + if box is pd.DataFrame: + assert result is NotImplemented + else: + tm.assert_equal(result, expected) # TODO: Should we be parametrizing over types for `ser` too? def test_float_series_rdiv_td64arr(self, box_with_array, names): diff --git a/pandas/tests/frame/test_cumulative.py b/pandas/tests/frame/test_cumulative.py index 5441505a920db..646b6d6ab3f1b 100644 --- a/pandas/tests/frame/test_cumulative.py +++ b/pandas/tests/frame/test_cumulative.py @@ -9,7 +9,7 @@ import numpy as np import pytest -from pandas import DataFrame, Series, _is_numpy_dev +from pandas import DataFrame, Series, _np_version_under1p18 import pandas._testing as tm @@ -75,7 +75,8 @@ def test_cumprod(self, datetime_frame): df.cumprod(1) @pytest.mark.xfail( - _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", + not _np_version_under1p18, + reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummin(self, datetime_frame): datetime_frame.iloc[5:10, 0] = np.nan @@ -101,7 +102,8 @@ def test_cummin(self, datetime_frame): assert np.shape(cummin_xs) == np.shape(datetime_frame) @pytest.mark.xfail( - _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", + not _np_version_under1p18, + reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummax(self, datetime_frame): datetime_frame.iloc[5:10, 0] = np.nan diff --git a/pandas/tests/series/test_cumulative.py b/pandas/tests/series/test_cumulative.py index e1a1f093f08b7..68de8f8e99aa9 100644 --- a/pandas/tests/series/test_cumulative.py +++ b/pandas/tests/series/test_cumulative.py @@ -11,7 +11,7 @@ import pytest import pandas as pd -from pandas import _is_numpy_dev +from pandas import _np_version_under1p18 import pandas._testing as tm @@ -39,7 +39,8 @@ def test_cumprod(self, datetime_series): _check_accum_op("cumprod", datetime_series) @pytest.mark.xfail( - _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", + not _np_version_under1p18, + reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummin(self, datetime_series): tm.assert_numpy_array_equal( @@ -55,7 +56,8 @@ def test_cummin(self, datetime_series): tm.assert_series_equal(result, expected) @pytest.mark.xfail( - _is_numpy_dev, reason="https://github.com/pandas-dev/pandas/issues/31992", + not _np_version_under1p18, + reason="https://github.com/pandas-dev/pandas/issues/31992", ) def test_cummax(self, datetime_series): tm.assert_numpy_array_equal( From 45b3cbbdbb22031e03a9c558e9618063ce13a2cc Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 28 Apr 2020 17:54:20 -0500 Subject: [PATCH 10/28] CI: Skip permissions test when running as sudo (#33847) --- pandas/tests/io/parser/test_common.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pandas/tests/io/parser/test_common.py b/pandas/tests/io/parser/test_common.py index 5bf9587a6ca22..55256499c6bb2 100644 --- a/pandas/tests/io/parser/test_common.py +++ b/pandas/tests/io/parser/test_common.py @@ -974,6 +974,15 @@ def test_no_permission(all_parsers): msg = r"\[Errno 13\]" with tm.ensure_clean() as path: os.chmod(path, 0) # make file unreadable + + # verify that this process cannot open the file (not running as sudo) + try: + with open(path): + pass + pytest.skip("Running as sudo.") + except PermissionError: + pass + with pytest.raises(PermissionError, match=msg) as e: parser.read_csv(path) assert path == e.value.filename From 1ce55895a743971a7122f59ed3d70d036108b670 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 28 Apr 2020 16:20:58 -0700 Subject: [PATCH 11/28] troubleshoot --- pandas/tests/frame/test_cumulative.py | 11 +---------- pandas/tests/groupby/test_categorical.py | 7 +++++-- pandas/tests/series/test_cumulative.py | 9 --------- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/pandas/tests/frame/test_cumulative.py b/pandas/tests/frame/test_cumulative.py index 646b6d6ab3f1b..248f3500c41df 100644 --- a/pandas/tests/frame/test_cumulative.py +++ b/pandas/tests/frame/test_cumulative.py @@ -7,9 +7,8 @@ """ import numpy as np -import pytest -from pandas import DataFrame, Series, _np_version_under1p18 +from pandas import DataFrame, Series import pandas._testing as tm @@ -74,10 +73,6 @@ def test_cumprod(self, datetime_frame): df.cumprod(0) df.cumprod(1) - @pytest.mark.xfail( - not _np_version_under1p18, - reason="https://github.com/pandas-dev/pandas/issues/31992", - ) def test_cummin(self, datetime_frame): datetime_frame.iloc[5:10, 0] = np.nan datetime_frame.iloc[10:15, 1] = np.nan @@ -101,10 +96,6 @@ def test_cummin(self, datetime_frame): cummin_xs = datetime_frame.cummin(axis=1) assert np.shape(cummin_xs) == np.shape(datetime_frame) - @pytest.mark.xfail( - not _np_version_under1p18, - reason="https://github.com/pandas-dev/pandas/issues/31992", - ) def test_cummax(self, datetime_frame): datetime_frame.iloc[5:10, 0] = np.nan datetime_frame.iloc[10:15, 1] = np.nan diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index ee0af141d0f95..781625f1f2ec9 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -3,7 +3,7 @@ import numpy as np import pytest -from pandas.compat import PY37 +from pandas.compat import PY37, is_platform_windows import pandas as pd from pandas import ( @@ -211,7 +211,10 @@ def test_level_get_group(observed): # GH#21636 flaky on py37; may be related to older numpy, see discussion # https://github.com/MacPython/pandas-wheels/pull/64 -@pytest.mark.xfail(PY37 and _np_version_under1p17, reason="Flaky, GH-27902") +@pytest.mark.xfail( + PY37 and _np_version_under1p17 and not is_platform_windows(), + reason="Flaky, GH-27902", +) @pytest.mark.parametrize("ordered", [True, False]) def test_apply(ordered): # GH 10138 diff --git a/pandas/tests/series/test_cumulative.py b/pandas/tests/series/test_cumulative.py index 68de8f8e99aa9..0b4c5f091106a 100644 --- a/pandas/tests/series/test_cumulative.py +++ b/pandas/tests/series/test_cumulative.py @@ -11,7 +11,6 @@ import pytest import pandas as pd -from pandas import _np_version_under1p18 import pandas._testing as tm @@ -38,10 +37,6 @@ def test_cumsum(self, datetime_series): def test_cumprod(self, datetime_series): _check_accum_op("cumprod", datetime_series) - @pytest.mark.xfail( - not _np_version_under1p18, - reason="https://github.com/pandas-dev/pandas/issues/31992", - ) def test_cummin(self, datetime_series): tm.assert_numpy_array_equal( datetime_series.cummin().values, @@ -55,10 +50,6 @@ def test_cummin(self, datetime_series): result.index = result.index._with_freq(None) tm.assert_series_equal(result, expected) - @pytest.mark.xfail( - not _np_version_under1p18, - reason="https://github.com/pandas-dev/pandas/issues/31992", - ) def test_cummax(self, datetime_series): tm.assert_numpy_array_equal( datetime_series.cummax().values, From 3fe8ab30e7344d3cd50121bc1f062a840d0ef52b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 00:18:30 -0700 Subject: [PATCH 12/28] TST: stop skipping Categorical take tests (#33859) --- pandas/tests/extension/test_categorical.py | 39 ---------------------- 1 file changed, 39 deletions(-) diff --git a/pandas/tests/extension/test_categorical.py b/pandas/tests/extension/test_categorical.py index 91d7cea90e0b3..d1211e477fe3e 100644 --- a/pandas/tests/extension/test_categorical.py +++ b/pandas/tests/extension/test_categorical.py @@ -97,8 +97,6 @@ class TestReshaping(base.BaseReshapingTests): class TestGetitem(base.BaseGetitemTests): - skip_take = pytest.mark.skip(reason="GH-20664.") - @pytest.mark.skip(reason="Backwards compatibility") def test_getitem_scalar(self, data): # CategoricalDtype.type isn't "correct" since it should @@ -106,43 +104,6 @@ def test_getitem_scalar(self, data): # to break things by changing. super().test_getitem_scalar(data) - @skip_take - def test_take(self, data, na_value, na_cmp): - # TODO remove this once Categorical.take is fixed - super().test_take(data, na_value, na_cmp) - - @skip_take - def test_take_negative(self, data): - super().test_take_negative(data) - - @skip_take - def test_take_pandas_style_negative_raises(self, data, na_value): - super().test_take_pandas_style_negative_raises(data, na_value) - - @skip_take - def test_take_non_na_fill_value(self, data_missing): - super().test_take_non_na_fill_value(data_missing) - - @skip_take - def test_take_out_of_bounds_raises(self, data, allow_fill): - return super().test_take_out_of_bounds_raises(data, allow_fill) - - @pytest.mark.skip(reason="GH-20747. Unobserved categories.") - def test_take_series(self, data): - super().test_take_series(data) - - @skip_take - def test_reindex_non_na_fill_value(self, data_missing): - super().test_reindex_non_na_fill_value(data_missing) - - @pytest.mark.skip(reason="Categorical.take buggy") - def test_take_empty(self, data, na_value, na_cmp): - super().test_take_empty(data, na_value, na_cmp) - - @pytest.mark.skip(reason="test not written correctly for categorical") - def test_reindex(self, data, na_value): - super().test_reindex(data, na_value) - class TestSetitem(base.BaseSetitemTests): pass From a2d025eef0b4f6f580924391a3b075ad83cc82eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Taljaard?= Date: Wed, 29 Apr 2020 11:32:07 +0200 Subject: [PATCH 13/28] Fix typo (#33861) --- pandas/core/arrays/datetimelike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index c325ec0d0bf7c..9d3ec284a2569 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1066,7 +1066,7 @@ def freqstr(self): @property # NB: override with cache_readonly in immutable subclasses def inferred_freq(self): """ - Tryies to return a string representing a frequency guess, + Tries to return a string representing a frequency guess, generated by infer_freq. Returns None if it can't autodetect the frequency. """ From 82e246383de733b136ee1330053c26715f4f1b67 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <2658661+dsaxton@users.noreply.github.com> Date: Wed, 29 Apr 2020 07:09:19 -0500 Subject: [PATCH 14/28] REF/CLN: Parametrize _isna (#33835) --- pandas/core/dtypes/missing.py | 70 ++++++++++++++--------------------- 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index 92e1b17c41694..87ff9a78909dd 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -1,6 +1,8 @@ """ missing types & inference """ +from functools import partial + import numpy as np from pandas._config import get_option @@ -124,61 +126,44 @@ def isna(obj): isnull = isna -def _isna_new(obj): - - if is_scalar(obj): - return libmissing.checknull(obj) - # hack (for now) because MI registers as ndarray - elif isinstance(obj, ABCMultiIndex): - raise NotImplementedError("isna is not defined for MultiIndex") - elif isinstance(obj, type): - return False - elif isinstance(obj, (ABCSeries, np.ndarray, ABCIndexClass, ABCExtensionArray)): - return _isna_ndarraylike(obj, old=False) - elif isinstance(obj, ABCDataFrame): - return obj.isna() - elif isinstance(obj, list): - return _isna_ndarraylike(np.asarray(obj, dtype=object), old=False) - elif hasattr(obj, "__array__"): - return _isna_ndarraylike(np.asarray(obj), old=False) - else: - return False - - -def _isna_old(obj): +def _isna(obj, inf_as_na: bool = False): """ - Detect missing values, treating None, NaN, INF, -INF as null. + Detect missing values, treating None, NaN or NA as null. Infinite + values will also be treated as null if inf_as_na is True. Parameters ---------- - arr: ndarray or object value + obj: ndarray or object value + Input array or scalar value. + inf_as_na: bool + Whether to treat infinity as null. Returns ------- boolean ndarray or boolean """ if is_scalar(obj): - return libmissing.checknull_old(obj) + if inf_as_na: + return libmissing.checknull_old(obj) + else: + return libmissing.checknull(obj) # hack (for now) because MI registers as ndarray elif isinstance(obj, ABCMultiIndex): raise NotImplementedError("isna is not defined for MultiIndex") elif isinstance(obj, type): return False elif isinstance(obj, (ABCSeries, np.ndarray, ABCIndexClass, ABCExtensionArray)): - return _isna_ndarraylike(obj, old=True) + return _isna_ndarraylike(obj, inf_as_na=inf_as_na) elif isinstance(obj, ABCDataFrame): return obj.isna() elif isinstance(obj, list): - return _isna_ndarraylike(np.asarray(obj, dtype=object), old=True) + return _isna_ndarraylike(np.asarray(obj, dtype=object), inf_as_na=inf_as_na) elif hasattr(obj, "__array__"): - return _isna_ndarraylike(np.asarray(obj), old=True) + return _isna_ndarraylike(np.asarray(obj), inf_as_na=inf_as_na) else: return False -_isna = _isna_new - - def _use_inf_as_na(key): """ Option change callback for na/inf behaviour. @@ -200,14 +185,11 @@ def _use_inf_as_na(key): * https://stackoverflow.com/questions/4859217/ programmatically-creating-variables-in-python/4859312#4859312 """ - flag = get_option(key) - if flag: - globals()["_isna"] = _isna_old - else: - globals()["_isna"] = _isna_new + inf_as_na = get_option(key) + globals()["_isna"] = partial(_isna, inf_as_na=inf_as_na) -def _isna_ndarraylike(obj, old: bool = False): +def _isna_ndarraylike(obj, inf_as_na: bool = False): """ Return an array indicating which values of the input array are NaN / NA. @@ -215,7 +197,7 @@ def _isna_ndarraylike(obj, old: bool = False): ---------- obj: array-like The input array whose elements are to be checked. - old: bool + inf_as_na: bool Whether or not to treat infinite values as NA. Returns @@ -227,17 +209,17 @@ def _isna_ndarraylike(obj, old: bool = False): dtype = values.dtype if is_extension_array_dtype(dtype): - if old: + if inf_as_na: result = values.isna() | (values == -np.inf) | (values == np.inf) else: result = values.isna() elif is_string_dtype(dtype): - result = _isna_string_dtype(values, dtype, old=old) + result = _isna_string_dtype(values, dtype, inf_as_na=inf_as_na) elif needs_i8_conversion(dtype): # this is the NaT pattern result = values.view("i8") == iNaT else: - if old: + if inf_as_na: result = ~np.isfinite(values) else: result = np.isnan(values) @@ -249,7 +231,9 @@ def _isna_ndarraylike(obj, old: bool = False): return result -def _isna_string_dtype(values: np.ndarray, dtype: np.dtype, old: bool) -> np.ndarray: +def _isna_string_dtype( + values: np.ndarray, dtype: np.dtype, inf_as_na: bool +) -> np.ndarray: # Working around NumPy ticket 1542 shape = values.shape @@ -257,7 +241,7 @@ def _isna_string_dtype(values: np.ndarray, dtype: np.dtype, old: bool) -> np.nda result = np.zeros(values.shape, dtype=bool) else: result = np.empty(shape, dtype=bool) - if old: + if inf_as_na: vec = libmissing.isnaobj_old(values.ravel()) else: vec = libmissing.isnaobj(values.ravel()) From 306044b8f250a3171bac867224d2b02497c2e577 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 29 Apr 2020 13:13:06 +0100 Subject: [PATCH 15/28] DOC: Code Standards Broken Link (#33865) --- doc/source/development/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development/contributing.rst b/doc/source/development/contributing.rst index ba7f7eb907f4a..1073959de4456 100644 --- a/doc/source/development/contributing.rst +++ b/doc/source/development/contributing.rst @@ -581,7 +581,7 @@ do not make sudden changes to the code that could have the potential to break a lot of user code as a result, that is, we need it to be as *backwards compatible* as possible to avoid mass breakages. -Additional standards are outlined on the `pandas code style guide `_ +Additional standards are outlined on the :ref:`pandas code style guide ` Optional dependencies --------------------- From c6845da9899e989b8f42bca1a4b6afbfa5a01823 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 29 Apr 2020 16:29:34 +0100 Subject: [PATCH 16/28] TYP/CLN: remove #type: ignore from pandas\tests\base\test_conversion.py (#33864) --- pandas/tests/base/test_conversion.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/tests/base/test_conversion.py b/pandas/tests/base/test_conversion.py index 59f9103072fe9..1f4bc024e14d4 100644 --- a/pandas/tests/base/test_conversion.py +++ b/pandas/tests/base/test_conversion.py @@ -391,10 +391,11 @@ def test_to_numpy_dtype(as_series): ), ], ) -@pytest.mark.parametrize("container", [pd.Series, pd.Index]) # type: ignore -def test_to_numpy_na_value_numpy_dtype(container, values, dtype, na_value, expected): - s = container(values) - result = s.to_numpy(dtype=dtype, na_value=na_value) +def test_to_numpy_na_value_numpy_dtype( + index_or_series, values, dtype, na_value, expected +): + obj = index_or_series(values) + result = obj.to_numpy(dtype=dtype, na_value=na_value) expected = np.array(expected) tm.assert_numpy_array_equal(result, expected) From f898f4ba549a5385d57a14b3cefced6bf5040571 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 12:11:49 -0700 Subject: [PATCH 17/28] CLN: address FIXME/TODO/XXX comments (#33858) --- ci/setup_env.sh | 2 +- pandas/_libs/src/parser/tokenizer.h | 4 ++-- pandas/_libs/tslibs/strptime.pyx | 2 +- pandas/core/algorithms.py | 1 + pandas/core/arrays/sparse/array.py | 2 +- pandas/core/dtypes/concat.py | 1 + pandas/core/groupby/generic.py | 1 + pandas/core/groupby/ops.py | 6 +++--- pandas/core/internals/blocks.py | 2 +- pandas/core/internals/concat.py | 1 + pandas/io/parsers.py | 2 +- pandas/tests/arithmetic/test_datetime64.py | 12 ++++-------- pandas/tests/arithmetic/test_period.py | 5 ++--- pandas/tests/arrays/boolean/test_logical.py | 1 + pandas/tests/base/test_conversion.py | 4 ---- pandas/tests/extension/base/dtype.py | 2 +- pandas/tests/extension/base/getitem.py | 3 ++- pandas/tests/extension/test_boolean.py | 1 + pandas/tests/frame/methods/test_round.py | 5 ----- pandas/tests/indexes/common.py | 3 +-- pandas/tests/indexes/datetimes/test_date_range.py | 4 ++-- pandas/tests/indexing/test_chaining_and_caching.py | 3 --- pandas/tests/io/excel/test_style.py | 8 ++++---- pandas/tests/io/formats/test_css.py | 2 -- pandas/tests/io/test_sql.py | 2 +- pandas/tests/reshape/test_concat.py | 2 +- pandas/tests/scalar/timedelta/test_arithmetic.py | 1 - pandas/tests/series/indexing/test_datetime.py | 2 -- pandas/tests/series/indexing/test_indexing.py | 2 +- pandas/tseries/offsets.py | 2 +- 30 files changed, 36 insertions(+), 52 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index ae39b0dda5d09..16bbd7693da37 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -128,7 +128,7 @@ conda list pandas echo "[Build extensions]" python setup.py build_ext -q -i -j2 -# XXX: Some of our environments end up with old versions of pip (10.x) +# TODO: Some of our environments end up with old versions of pip (10.x) # Adding a new enough version of pip to the requirements explodes the # solve time. Just using pip to update itself. # - py35_macos diff --git a/pandas/_libs/src/parser/tokenizer.h b/pandas/_libs/src/parser/tokenizer.h index 4fd2065c07100..7dfae737718a5 100644 --- a/pandas/_libs/src/parser/tokenizer.h +++ b/pandas/_libs/src/parser/tokenizer.h @@ -52,8 +52,8 @@ See LICENSE for the license #define PARSER_OUT_OF_MEMORY -1 /* - * XXX Might want to couple count_rows() with read_rows() to avoid duplication - * of some file I/O. + * TODO: Might want to couple count_rows() with read_rows() to avoid + * duplication of some file I/O. */ typedef enum { diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index ce4d3a4ef8e02..c56d24d68a175 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -544,7 +544,7 @@ class TimeRE(dict): 'w': r"(?P[0-6])", # W is set below by using 'U' 'y': r"(?P\d\d)", - # XXX: Does 'Y' need to worry about having less or more than + # TODO: Does 'Y' need to worry about having less or more than # 4 digits? 'Y': r"(?P\d\d\d\d)", 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)", diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index eca1733b61a52..ce22c3f5416fa 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -142,6 +142,7 @@ def _ensure_data(values, dtype=None): if values.ndim > 1 and is_datetime64_ns_dtype(values): # Avoid calling the DatetimeIndex constructor as it is 1D only # Note: this is reached by DataFrame.rank calls GH#27027 + # TODO(EA2D): special case not needed with 2D EAs asi8 = values.view("i8") dtype = values.dtype return asi8, dtype diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index 620e157ee54ec..72b6e07942d5e 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -335,7 +335,7 @@ def __init__( # TODO: disentangle the fill_value dtype inference from # dtype inference if data is None: - # XXX: What should the empty dtype be? Object or float? + # TODO: What should the empty dtype be? Object or float? data = np.array([], dtype=dtype) if not is_array_like(data): diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py index 257c4fe3c6d30..2c560a1ed8c62 100644 --- a/pandas/core/dtypes/concat.py +++ b/pandas/core/dtypes/concat.py @@ -178,6 +178,7 @@ def concat_categorical(to_concat, axis: int = 0): ] result = concat_compat(to_concat) if axis == 1: + # TODO(EA2D): not necessary with 2D EAs result = result.reshape(1, len(result)) return result diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index ddf553dd1dd62..6745203d5beb7 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -1129,6 +1129,7 @@ def _cython_agg_blocks( # e.g. block.values was an IntegerArray # (1, N) case can occur if block.values was Categorical # and result is ndarray[object] + # TODO(EA2D): special casing not needed with 2D EAs assert result.ndim == 1 or result.shape[0] == 1 try: # Cast back if feasible diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py index 65788970628dc..f799baf354794 100644 --- a/pandas/core/groupby/ops.py +++ b/pandas/core/groupby/ops.py @@ -476,8 +476,8 @@ def _cython_operation( if is_datetime64tz_dtype(values.dtype): # Cast to naive; we'll cast back at the end of the function - # TODO: possible need to reshape? kludge can be avoided when - # 2D EA is allowed. + # TODO: possible need to reshape? + # TODO(EA2D):kludge can be avoided when 2D EA is allowed. values = values.view("M8[ns]") is_datetimelike = needs_i8_conversion(values.dtype) @@ -717,7 +717,7 @@ def _aggregate_series_pure_python( if isinstance(res, (Series, Index, np.ndarray)): if len(res) == 1: # e.g. test_agg_lambda_with_timezone lambda e: e.head(1) - # FIXME: are we potentially losing import res.index info? + # FIXME: are we potentially losing important res.index info? res = res.item() else: raise ValueError("Function does not reduce") diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 366ea54a510ef..c40fad6b434d0 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1721,7 +1721,7 @@ def take_nd( return self.make_block_same_class(new_values, new_mgr_locs) def _can_hold_element(self, element: Any) -> bool: - # XXX: We may need to think about pushing this onto the array. + # TODO: We may need to think about pushing this onto the array. # We're doing the same as CategoricalBlock here. return True diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index 743dd6db348b4..e9bbd915df768 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -218,6 +218,7 @@ def is_na(self): elif is_sparse(self.block.values.dtype): return False elif self.block.is_extension: + # TODO(EA2D): no need for special case with 2D EAs values_flat = values else: values_flat = values.ravel(order="K") diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 2df81ba0aa51a..f289db39347ae 100644 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -1851,7 +1851,7 @@ def _cast_types(self, values, cast_type, column): ) if not is_object_dtype(values) and not known_cats: - # XXX this is for consistency with + # TODO: this is for consistency with # c-parser which parses all categories # as strings values = astype_nansafe(values, str) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 6e116f56d1494..8c480faa4ee81 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -140,12 +140,10 @@ def test_dt64arr_nat_comparison(self, tz_naive_fixture, box_with_array): ts = pd.Timestamp.now(tz) ser = pd.Series([ts, pd.NaT]) - # FIXME: Can't transpose because that loses the tz dtype on - # the NaT column - obj = tm.box_expected(ser, box, transpose=False) + obj = tm.box_expected(ser, box) expected = pd.Series([True, False], dtype=np.bool_) - expected = tm.box_expected(expected, xbox, transpose=False) + expected = tm.box_expected(expected, xbox) result = obj == ts tm.assert_equal(result, expected) @@ -844,10 +842,8 @@ def test_dt64arr_add_sub_td64_nat(self, box_with_array, tz_naive_fixture): other = np.timedelta64("NaT") expected = pd.DatetimeIndex(["NaT"] * 9, tz=tz) - # FIXME: fails with transpose=True due to tz-aware DataFrame - # transpose bug - obj = tm.box_expected(dti, box_with_array, transpose=False) - expected = tm.box_expected(expected, box_with_array, transpose=False) + obj = tm.box_expected(dti, box_with_array) + expected = tm.box_expected(expected, box_with_array) result = obj + other tm.assert_equal(result, expected) diff --git a/pandas/tests/arithmetic/test_period.py b/pandas/tests/arithmetic/test_period.py index 55a547b361eb3..0e8bec331ea3f 100644 --- a/pandas/tests/arithmetic/test_period.py +++ b/pandas/tests/arithmetic/test_period.py @@ -906,9 +906,8 @@ def test_pi_add_offset_n_gt1_not_divisible(self, box_with_array): pi = pd.PeriodIndex(["2016-01"], freq="2M") expected = pd.PeriodIndex(["2016-04"], freq="2M") - # FIXME: with transposing these tests fail - pi = tm.box_expected(pi, box_with_array, transpose=False) - expected = tm.box_expected(expected, box_with_array, transpose=False) + pi = tm.box_expected(pi, box_with_array) + expected = tm.box_expected(expected, box_with_array) result = pi + to_offset("3M") tm.assert_equal(result, expected) diff --git a/pandas/tests/arrays/boolean/test_logical.py b/pandas/tests/arrays/boolean/test_logical.py index 6cfe19e2fe3eb..bf4775bbd7b32 100644 --- a/pandas/tests/arrays/boolean/test_logical.py +++ b/pandas/tests/arrays/boolean/test_logical.py @@ -38,6 +38,7 @@ def test_empty_ok(self, all_logical_operators): result = getattr(a, op_name)(False) tm.assert_extension_array_equal(a, result) + # FIXME: dont leave commented-out # TODO: pd.NA # result = getattr(a, op_name)(pd.NA) # tm.assert_extension_array_equal(a, result) diff --git a/pandas/tests/base/test_conversion.py b/pandas/tests/base/test_conversion.py index 1f4bc024e14d4..e737a09b2ec6d 100644 --- a/pandas/tests/base/test_conversion.py +++ b/pandas/tests/base/test_conversion.py @@ -48,8 +48,6 @@ class TestToIterable: ], ids=["tolist", "to_list", "list", "iter"], ) - @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") - # TODO(GH-24559): Remove the filterwarnings def test_iterable(self, index_or_series, method, dtype, rdtype): # gh-10904 # gh-13258 @@ -104,8 +102,6 @@ def test_iterable_items(self, dtype, rdtype): @pytest.mark.parametrize( "dtype, rdtype", dtypes + [("object", int), ("category", int)] ) - @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") - # TODO(GH-24559): Remove the filterwarnings def test_iterable_map(self, index_or_series, dtype, rdtype): # gh-13236 # coerce iteration to underlying python / pandas types diff --git a/pandas/tests/extension/base/dtype.py b/pandas/tests/extension/base/dtype.py index b01867624cb16..ee4e199fbfe45 100644 --- a/pandas/tests/extension/base/dtype.py +++ b/pandas/tests/extension/base/dtype.py @@ -75,7 +75,7 @@ def test_check_dtype(self, data): else: expected = pd.Series([True, True, False, False], index=list("ABCD")) - # XXX: This should probably be *fixed* not ignored. + # FIXME: This should probably be *fixed* not ignored. # See libops.scalar_compare with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) diff --git a/pandas/tests/extension/base/getitem.py b/pandas/tests/extension/base/getitem.py index dc94bffd320b1..251376798efc3 100644 --- a/pandas/tests/extension/base/getitem.py +++ b/pandas/tests/extension/base/getitem.py @@ -230,7 +230,8 @@ def test_getitem_integer_with_missing_raises(self, data, idx): with pytest.raises(ValueError, match=msg): data[idx] - # TODO this raises KeyError about labels not found (it tries label-based) + # FIXME: dont leave commented-out + # TODO: this raises KeyError about labels not found (it tries label-based) # import pandas._testing as tm # s = pd.Series(data, index=[tm.rands(4) for _ in range(len(data))]) # with pytest.raises(ValueError, match=msg): diff --git a/pandas/tests/extension/test_boolean.py b/pandas/tests/extension/test_boolean.py index 303764bff50b6..0b8b334e53d68 100644 --- a/pandas/tests/extension/test_boolean.py +++ b/pandas/tests/extension/test_boolean.py @@ -356,6 +356,7 @@ class TestUnaryOps(base.BaseUnaryOpsTests): pass +# FIXME: dont leave commented-out # TODO parsing not yet supported # class TestParsing(base.BaseParsingTests): # pass diff --git a/pandas/tests/frame/methods/test_round.py b/pandas/tests/frame/methods/test_round.py index 6dcdf49e93729..3051f27882fb8 100644 --- a/pandas/tests/frame/methods/test_round.py +++ b/pandas/tests/frame/methods/test_round.py @@ -102,11 +102,6 @@ def test_round(self): # nan in Series round nan_round_Series = Series({"col1": np.nan, "col2": 1}) - # TODO(wesm): unused? - expected_nan_round = DataFrame( # noqa - {"col1": [1.123, 2.123, 3.123], "col2": [1.2, 2.2, 3.2]} - ) - msg = "integer argument expected, got float" with pytest.raises(TypeError, match=msg): df.round(nan_round_Series) diff --git a/pandas/tests/indexes/common.py b/pandas/tests/indexes/common.py index 52b82b36d13be..8e8f4738d30dd 100644 --- a/pandas/tests/indexes/common.py +++ b/pandas/tests/indexes/common.py @@ -561,8 +561,7 @@ def test_equals(self, indices): assert not indices.equals(np.array(indices)) # Cannot pass in non-int64 dtype to RangeIndex - if not isinstance(indices, (RangeIndex, CategoricalIndex)): - # TODO: CategoricalIndex can be re-allowed following GH#32167 + if not isinstance(indices, RangeIndex): same_values = Index(indices, dtype=object) assert indices.equals(same_values) assert same_values.equals(indices) diff --git a/pandas/tests/indexes/datetimes/test_date_range.py b/pandas/tests/indexes/datetimes/test_date_range.py index 6ddbe4a5ce0a5..d46fe0181ea1b 100644 --- a/pandas/tests/indexes/datetimes/test_date_range.py +++ b/pandas/tests/indexes/datetimes/test_date_range.py @@ -846,7 +846,7 @@ def test_daterange_bug_456(self): # GH #456 rng1 = bdate_range("12/5/2011", "12/5/2011") rng2 = bdate_range("12/2/2011", "12/5/2011") - rng2._data.freq = BDay() # TODO: shouldn't this already be set? + assert rng2._data.freq == BDay() result = rng1.union(rng2) assert isinstance(result, DatetimeIndex) @@ -905,7 +905,7 @@ def test_daterange_bug_456(self): # GH #456 rng1 = bdate_range("12/5/2011", "12/5/2011", freq="C") rng2 = bdate_range("12/2/2011", "12/5/2011", freq="C") - rng2._data.freq = CDay() # TODO: shouldn't this already be set? + assert rng2._data.freq == CDay() result = rng1.union(rng2) assert isinstance(result, DatetimeIndex) diff --git a/pandas/tests/indexing/test_chaining_and_caching.py b/pandas/tests/indexing/test_chaining_and_caching.py index 17722e949df1e..fa5fe5ba5c384 100644 --- a/pandas/tests/indexing/test_chaining_and_caching.py +++ b/pandas/tests/indexing/test_chaining_and_caching.py @@ -376,9 +376,6 @@ def test_cache_updating(self): df["f"] = 0 df.f.values[3] = 1 - # TODO(wesm): unused? - # y = df.iloc[np.arange(2, len(df))] - df.f.values[3] = 2 expected = DataFrame( np.zeros((5, 6), dtype="int64"), diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index 31b033f381f0c..936fc175a493b 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -23,7 +23,7 @@ ) def test_styler_to_excel(engine): def style(df): - # XXX: RGB colors not supported in xlwt + # TODO: RGB colors not supported in xlwt return DataFrame( [ ["font-weight: bold", "", ""], @@ -47,7 +47,7 @@ def assert_equal_style(cell1, cell2, engine): pytest.xfail( reason=(f"GH25351: failing on some attribute comparisons in {engine}") ) - # XXX: should find a better way to check equality + # TODO: should find a better way to check equality assert cell1.alignment.__dict__ == cell2.alignment.__dict__ assert cell1.border.__dict__ == cell2.border.__dict__ assert cell1.fill.__dict__ == cell2.fill.__dict__ @@ -98,7 +98,7 @@ def custom_converter(css): # (2) check styling with default converter - # XXX: openpyxl (as at 2.4) prefixes colors with 00, xlsxwriter with FF + # TODO: openpyxl (as at 2.4) prefixes colors with 00, xlsxwriter with FF alpha = "00" if engine == "openpyxl" else "FF" n_cells = 0 @@ -106,7 +106,7 @@ def custom_converter(css): assert len(col1) == len(col2) for cell1, cell2 in zip(col1, col2): ref = f"{cell2.column}{cell2.row:d}" - # XXX: this isn't as strong a test as ideal; we should + # TODO: this isn't as strong a test as ideal; we should # confirm that differences are exclusive if ref == "B2": assert not cell1.font.bold diff --git a/pandas/tests/io/formats/test_css.py b/pandas/tests/io/formats/test_css.py index 7008cef7b28fa..9383f86e335fa 100644 --- a/pandas/tests/io/formats/test_css.py +++ b/pandas/tests/io/formats/test_css.py @@ -60,8 +60,6 @@ def test_css_parse_invalid(invalid_css, remainder): with tm.assert_produces_warning(CSSWarning): assert_same_resolution(invalid_css, remainder) - # TODO: we should be checking that in other cases no warnings are raised - @pytest.mark.parametrize( "shorthand,expansions", diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 70f3f99442183..8c424e07601b8 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1130,7 +1130,7 @@ def setup_method(self, load_iris_data): self.conn.close() self.conn = self.__engine self.pandasSQL = sql.SQLDatabase(self.__engine) - # XXX: + # FIXME: dont leave commented-out # super().teardown_method(method) diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py index c4025640bb49f..7c01664df0607 100644 --- a/pandas/tests/reshape/test_concat.py +++ b/pandas/tests/reshape/test_concat.py @@ -228,7 +228,7 @@ def test_concatlike_dtypes_coercion(self): # same dtype is tested in test_concatlike_same_dtypes continue elif typ1 == "category" or typ2 == "category": - # ToDo: suspicious + # TODO: suspicious continue # specify expected dtype diff --git a/pandas/tests/scalar/timedelta/test_arithmetic.py b/pandas/tests/scalar/timedelta/test_arithmetic.py index 1663fa65c7dfe..17d6edfd17de8 100644 --- a/pandas/tests/scalar/timedelta/test_arithmetic.py +++ b/pandas/tests/scalar/timedelta/test_arithmetic.py @@ -651,7 +651,6 @@ def test_td_rfloordiv_numeric_series(self): msg = "Invalid dtype" with pytest.raises(TypeError, match=msg): # Deprecated GH#19761, enforced GH#29797 - # TODO: GH-19761. Change to TypeError. ser // td # ---------------------------------------------------------------- diff --git a/pandas/tests/series/indexing/test_datetime.py b/pandas/tests/series/indexing/test_datetime.py index 22ef966299d5b..0b34fab7b80b1 100644 --- a/pandas/tests/series/indexing/test_datetime.py +++ b/pandas/tests/series/indexing/test_datetime.py @@ -556,8 +556,6 @@ def test_indexing_unordered(): ts2 = pd.concat([ts[0:4], ts[-4:], ts[4:-4]]) for t in ts.index: - # TODO: unused? - s = str(t) # noqa expected = ts[t] result = ts2[t] diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index c2b5117d395f9..16b6604c7fc52 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -445,7 +445,7 @@ def test_setitem_with_tz(tz): def test_setitem_with_tz_dst(): - # GH XXX + # GH XXX TODO: fill in GH ref tz = "US/Eastern" orig = pd.Series(pd.date_range("2016-11-06", freq="H", periods=3, tz=tz)) assert orig.dtype == f"datetime64[ns, {tz}]" diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 5fe62f41e49ff..286ee91bc7d4f 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -435,7 +435,7 @@ def rollforward(self, dt): def is_on_offset(self, dt): if self.normalize and not _is_normalized(dt): return False - # XXX, see #1395 + # TODO, see #1395 if type(self) == DateOffset or isinstance(self, Tick): return True From ef09e64345a3515667e1c69c1e037935988f9b45 Mon Sep 17 00:00:00 2001 From: Kaiqi Dong Date: Wed, 29 Apr 2020 21:26:11 +0200 Subject: [PATCH 18/28] DOC: Remove TODO from `test_openpyxl` (#33872) * remove \n from docstring * fix issue 17038 * revert change * revert change * Remove TODO --- pandas/tests/io/excel/test_openpyxl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 60c943d95e510..5f8d58ea1f105 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -109,7 +109,6 @@ def test_write_append_mode(ext, mode, expected): def test_to_excel_with_openpyxl_engine(ext, tmpdir): # GH 29854 - # TODO: Fix this once newer version of openpyxl fixes the bug df1 = DataFrame({"A": np.linspace(1, 10, 10)}) df2 = DataFrame({"B": np.linspace(1, 20, 10)}) df = pd.concat([df1, df2], axis=1) From 7bf479b60b25072d1d37dbe191b8d04713e06c12 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 13:58:00 -0700 Subject: [PATCH 19/28] remove box_df_fail --- pandas/tests/arithmetic/conftest.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/pandas/tests/arithmetic/conftest.py b/pandas/tests/arithmetic/conftest.py index c20a9567e9ff8..8b9e5cd371a90 100644 --- a/pandas/tests/arithmetic/conftest.py +++ b/pandas/tests/arithmetic/conftest.py @@ -231,22 +231,6 @@ def box(request): return request.param -@pytest.fixture( - params=[ - pd.Index, - pd.Series, - pytest.param(pd.DataFrame, marks=pytest.mark.xfail), - tm.to_array, - ], - ids=id_func, -) -def box_df_fail(request): - """ - Fixture equivalent to `box` fixture but xfailing the DataFrame case. - """ - return request.param - - @pytest.fixture(params=[pd.Index, pd.Series, pd.DataFrame, tm.to_array], ids=id_func) def box_with_array(request): """ From 6863882c6e66ce45b72fccd06e28442c7b3e2ab4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 15:35:01 -0700 Subject: [PATCH 20/28] TST: cactch/suppress warnings --- pandas/tests/plotting/test_datetimelike.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/tests/plotting/test_datetimelike.py b/pandas/tests/plotting/test_datetimelike.py index b85a2affc4e4b..91dc9ee2f1c08 100644 --- a/pandas/tests/plotting/test_datetimelike.py +++ b/pandas/tests/plotting/test_datetimelike.py @@ -1242,7 +1242,9 @@ def test_ax_plot(self): x = date_range(start="2012-01-02", periods=10, freq="D") y = list(range(len(x))) _, ax = self.plt.subplots() - lines = ax.plot(x, y, label="Y") + with tm.assert_produces_warning(DeprecationWarning, check_stacklevel=False): + # multi-dimensional indexing + lines = ax.plot(x, y, label="Y") tm.assert_index_equal(DatetimeIndex(lines[0].get_xdata()), x) @pytest.mark.slow @@ -1456,7 +1458,9 @@ def test_add_matplotlib_datetime64(self): # ax.xaxis.converter with a DatetimeConverter s = Series(np.random.randn(10), index=date_range("1970-01-02", periods=10)) ax = s.plot() - ax.plot(s.index, s.values, color="g") + with tm.assert_produces_warning(DeprecationWarning): + # multi-dimensional indexing + ax.plot(s.index, s.values, color="g") l1, l2 = ax.lines tm.assert_numpy_array_equal(l1.get_xydata(), l2.get_xydata()) From 32666d9c996622d39f57c97c4becea6610d15d6f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 15:39:22 -0700 Subject: [PATCH 21/28] TST: catch/suppress warnings --- pandas/tests/io/test_common.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index 84bc29ebc65e0..b27b028694d20 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -235,6 +235,11 @@ def test_read_expands_user_home_dir( ), ], ) + @pytest.mark.filterwarnings( + "ignore:This method will be removed in future versions. " + r"Use 'tree.iter\(\)' or 'list\(tree.iter\(\)\)' instead." + ":PendingDeprecationWarning" + ) def test_read_fspath_all(self, reader, module, path, datapath): pytest.importorskip(module) path = datapath(*path) From f27775c6f48eaa932068494e3a01b5b3e2edb693 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 15:45:22 -0700 Subject: [PATCH 22/28] TST: dynamic xfails --- pandas/tests/arithmetic/test_interval.py | 15 ++++++++++----- pandas/tests/indexes/test_index_new.py | 5 +++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pandas/tests/arithmetic/test_interval.py b/pandas/tests/arithmetic/test_interval.py index d7c312b2fda1b..72e66e38cddd7 100644 --- a/pandas/tests/arithmetic/test_interval.py +++ b/pandas/tests/arithmetic/test_interval.py @@ -126,12 +126,15 @@ def test_compare_scalar_interval_mixed_closed(self, op, closed, other_closed): expected = self.elementwise_comparison(op, array, other) tm.assert_numpy_array_equal(result, expected) - def test_compare_scalar_na(self, op, array, nulls_fixture): + def test_compare_scalar_na(self, op, array, nulls_fixture, request): result = op(array, nulls_fixture) expected = self.elementwise_comparison(op, array, nulls_fixture) if nulls_fixture is pd.NA and array.dtype != pd.IntervalDtype("int"): - pytest.xfail("broken for non-integer IntervalArray; see GH 31882") + mark = pytest.mark.xfail( + reason="broken for non-integer IntervalArray; see GH 31882" + ) + request.node.add_marker(mark) tm.assert_numpy_array_equal(result, expected) @@ -207,13 +210,15 @@ def test_compare_list_like_object(self, op, array, other): expected = self.elementwise_comparison(op, array, other) tm.assert_numpy_array_equal(result, expected) - def test_compare_list_like_nan(self, op, array, nulls_fixture): + def test_compare_list_like_nan(self, op, array, nulls_fixture, request): other = [nulls_fixture] * 4 result = op(array, other) expected = self.elementwise_comparison(op, array, other) - if nulls_fixture is pd.NA: - pytest.xfail("broken for non-integer IntervalArray; see GH 31882") + if nulls_fixture is pd.NA and array.dtype.subtype != "i8": + reason = "broken for non-integer IntervalArray; see GH 31882" + mark = pytest.mark.xfail(reason=reason) + request.node.add_marker(mark) tm.assert_numpy_array_equal(result, expected) diff --git a/pandas/tests/indexes/test_index_new.py b/pandas/tests/indexes/test_index_new.py index 87df5959e6221..9248c185bccd7 100644 --- a/pandas/tests/indexes/test_index_new.py +++ b/pandas/tests/indexes/test_index_new.py @@ -83,7 +83,7 @@ def test_constructor_infer_periodindex(self): ], ) def test_constructor_infer_nat_dt_like( - self, pos, klass, dtype, ctor, nulls_fixture + self, pos, klass, dtype, ctor, nulls_fixture, request ): expected = klass([NaT, NaT]) assert expected.dtype == dtype @@ -92,7 +92,8 @@ def test_constructor_infer_nat_dt_like( if nulls_fixture is NA: expected = Index([NA, NaT]) - pytest.xfail("Broken with np.NaT ctor; see GH 31884") + mark = pytest.mark.xfail(reason="Broken with np.NaT ctor; see GH 31884") + request.node.add_marker(mark) result = Index(data) tm.assert_index_equal(result, expected) From 67e87313b973e143f28e0f34d2182de379d08184 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 29 Apr 2020 17:54:28 -0700 Subject: [PATCH 23/28] troubleshoot CI --- pandas/tests/arithmetic/test_interval.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_interval.py b/pandas/tests/arithmetic/test_interval.py index 72e66e38cddd7..5a283edee1ac9 100644 --- a/pandas/tests/arithmetic/test_interval.py +++ b/pandas/tests/arithmetic/test_interval.py @@ -3,6 +3,8 @@ import numpy as np import pytest +from pandas.compat import is_platform_windows + from pandas.core.dtypes.common import is_list_like import pandas as pd @@ -130,7 +132,11 @@ def test_compare_scalar_na(self, op, array, nulls_fixture, request): result = op(array, nulls_fixture) expected = self.elementwise_comparison(op, array, nulls_fixture) - if nulls_fixture is pd.NA and array.dtype != pd.IntervalDtype("int"): + if ( + nulls_fixture is pd.NA + and array.dtype != pd.IntervalDtype("int") + and not is_platform_windows() + ): mark = pytest.mark.xfail( reason="broken for non-integer IntervalArray; see GH 31882" ) From 650aed3f52a7fb93ac0f59269687878833c37376 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 30 Apr 2020 08:11:04 -0700 Subject: [PATCH 24/28] un-silence --- pandas/tests/plotting/test_datetimelike.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/tests/plotting/test_datetimelike.py b/pandas/tests/plotting/test_datetimelike.py index 91dc9ee2f1c08..7dcb692e29337 100644 --- a/pandas/tests/plotting/test_datetimelike.py +++ b/pandas/tests/plotting/test_datetimelike.py @@ -1242,9 +1242,7 @@ def test_ax_plot(self): x = date_range(start="2012-01-02", periods=10, freq="D") y = list(range(len(x))) _, ax = self.plt.subplots() - with tm.assert_produces_warning(DeprecationWarning, check_stacklevel=False): - # multi-dimensional indexing - lines = ax.plot(x, y, label="Y") + lines = ax.plot(x, y, label="Y") tm.assert_index_equal(DatetimeIndex(lines[0].get_xdata()), x) @pytest.mark.slow From 954994de92646f74ada38ff03089ce46794cfcde Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 30 Apr 2020 08:52:35 -0700 Subject: [PATCH 25/28] troubleshoot CI --- pandas/tests/arithmetic/test_interval.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pandas/tests/arithmetic/test_interval.py b/pandas/tests/arithmetic/test_interval.py index 5a283edee1ac9..72e66e38cddd7 100644 --- a/pandas/tests/arithmetic/test_interval.py +++ b/pandas/tests/arithmetic/test_interval.py @@ -3,8 +3,6 @@ import numpy as np import pytest -from pandas.compat import is_platform_windows - from pandas.core.dtypes.common import is_list_like import pandas as pd @@ -132,11 +130,7 @@ def test_compare_scalar_na(self, op, array, nulls_fixture, request): result = op(array, nulls_fixture) expected = self.elementwise_comparison(op, array, nulls_fixture) - if ( - nulls_fixture is pd.NA - and array.dtype != pd.IntervalDtype("int") - and not is_platform_windows() - ): + if nulls_fixture is pd.NA and array.dtype != pd.IntervalDtype("int"): mark = pytest.mark.xfail( reason="broken for non-integer IntervalArray; see GH 31882" ) From 33d2aab96ce581d073eaa1ccb340127f13d3575f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 30 Apr 2020 09:48:12 -0700 Subject: [PATCH 26/28] troubleshoot CI --- pandas/tests/io/pytables/test_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index 299ae2f41d676..2162bb6b399fc 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -1581,6 +1581,7 @@ def check_col(key, name, size): result = store.select("df_dc", ["B > 0", "C > 0", "string == foo"]) expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")] + expected.index = expected.index._with_freq(None) tm.assert_frame_equal(result, expected, check_index_type=False) with ensure_clean_store(setup_path) as store: From 864184342519cf25b19f91bbfc2cc12ed3970bcb Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 30 Apr 2020 11:21:01 -0700 Subject: [PATCH 27/28] revert --- pandas/tests/io/pytables/test_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index 2162bb6b399fc..299ae2f41d676 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -1581,7 +1581,6 @@ def check_col(key, name, size): result = store.select("df_dc", ["B > 0", "C > 0", "string == foo"]) expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")] - expected.index = expected.index._with_freq(None) tm.assert_frame_equal(result, expected, check_index_type=False) with ensure_clean_store(setup_path) as store: From 930a4215109548054e7f5b80b234843664eb5160 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 30 Apr 2020 12:45:17 -0700 Subject: [PATCH 28/28] troubleshoot interval --- pandas/tests/arithmetic/test_interval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_interval.py b/pandas/tests/arithmetic/test_interval.py index 72e66e38cddd7..66526204a6208 100644 --- a/pandas/tests/arithmetic/test_interval.py +++ b/pandas/tests/arithmetic/test_interval.py @@ -130,7 +130,7 @@ def test_compare_scalar_na(self, op, array, nulls_fixture, request): result = op(array, nulls_fixture) expected = self.elementwise_comparison(op, array, nulls_fixture) - if nulls_fixture is pd.NA and array.dtype != pd.IntervalDtype("int"): + if nulls_fixture is pd.NA and array.dtype != pd.IntervalDtype("int64"): mark = pytest.mark.xfail( reason="broken for non-integer IntervalArray; see GH 31882" )