From d6356206ab681f6732e1a7d270a689a667c69f08 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 10:09:58 -0500 Subject: [PATCH 1/7] TST: More maybe_promote xfails --- pandas/core/dtypes/cast.py | 15 ++++++++++++--- pandas/tests/dtypes/cast/test_promote.py | 11 ----------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 5801384bf8db9..1b5340468034a 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -348,12 +348,21 @@ def maybe_promote(dtype, fill_value=np.nan): dtype = np.dtype(np.object_) fill_value = np.nan + if dtype == np.object_ or dtype.kind in ["U", "S"]: + # We treat string-like dtypes as object, and _always_ fill + # with np.nan + fill_value = np.nan + # returns tuple of (dtype, fill_value) if issubclass(dtype.type, np.datetime64): - try: - fill_value = tslibs.Timestamp(fill_value).to_datetime64() - except (TypeError, ValueError): + if isinstance(fill_value, datetime) and fill_value.tzinfo is not None: + # Trying to insert tzaware into tznaive, have to cast to object dtype = np.dtype(np.object_) + else: + try: + fill_value = tslibs.Timestamp(fill_value).to_datetime64() + except (TypeError, ValueError): + dtype = np.dtype(np.object_) elif issubclass(dtype.type, np.timedelta64): try: fv = tslibs.Timedelta(fill_value) diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index 1b7de9b20f42f..d80004b674dbc 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -484,9 +484,6 @@ def test_maybe_promote_any_numpy_dtype_with_datetimetz( fill_dtype = DatetimeTZDtype(tz=tz_aware_fixture) boxed, box_dtype = box # read from parametrized fixture - if dtype.kind == "M" and not boxed: - pytest.xfail("Comes back as M8 instead of object") - fill_value = pd.Series([fill_value], dtype=fill_dtype)[0] # filling any numpy dtype with datetimetz casts to object @@ -572,11 +569,6 @@ def test_maybe_promote_string_with_any(string_dtype, any_numpy_dtype_reduced, bo fill_dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if boxed and box_dtype is None and fill_dtype.kind == "m": - pytest.xfail("wrong missing value marker") - if boxed and box_dtype is None and fill_dtype.kind == "M": - pytest.xfail("wrong missing value marker") - # create array of given dtype; casts "1" to correct dtype fill_value = np.array([1], dtype=fill_dtype)[0] @@ -639,9 +631,6 @@ def test_maybe_promote_object_with_any(object_dtype, any_numpy_dtype_reduced, bo fill_dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if boxed and box_dtype is None and is_datetime_or_timedelta_dtype(fill_dtype): - pytest.xfail("wrong missing value marker") - # create array of given dtype; casts "1" to correct dtype fill_value = np.array([1], dtype=fill_dtype)[0] From 677825b9b02c609a490631415866a929a9f2b2a4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 10:51:32 -0500 Subject: [PATCH 2/7] cleanup --- pandas/core/dtypes/cast.py | 3 +-- pandas/tests/dtypes/cast/test_promote.py | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 1b5340468034a..7bce86b572f0c 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -352,6 +352,7 @@ def maybe_promote(dtype, fill_value=np.nan): # We treat string-like dtypes as object, and _always_ fill # with np.nan fill_value = np.nan + dtype = np.dtype(np.object_) # returns tuple of (dtype, fill_value) if issubclass(dtype.type, np.datetime64): @@ -424,8 +425,6 @@ def maybe_promote(dtype, fill_value=np.nan): # in case we have a string that looked like a number if is_extension_array_dtype(dtype): pass - elif is_datetime64tz_dtype(dtype): - pass elif issubclass(np.dtype(dtype).type, (bytes, str)): dtype = np.object_ diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index d80004b674dbc..1881ce295a4a2 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -332,18 +332,17 @@ def test_maybe_promote_any_with_datetime64( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if is_datetime64_dtype(dtype): - if boxed and ( - box_dtype == object - or (box_dtype is None and not is_datetime64_dtype(type(fill_value))) - ): - pytest.xfail("falsely upcasts to object") - else: - if boxed and ( - box_dtype == "dt_dtype" - or (box_dtype is None and is_datetime64_dtype(type(fill_value))) - ): - pytest.xfail("mix of lack of upcasting, resp. wrong missing value") + if boxed: + if is_datetime64_dtype(dtype): + if box_dtype == object: + pytest.xfail("falsely upcasts to object") + elif box_dtype is None and not is_datetime64_dtype(type(fill_value)): + pytest.xfail("falsely upcasts to object") + else: + if box_dtype == "dt_dtype": + pytest.xfail("mix of lack of upcasting, resp. wrong missing value") + elif box_dtype is None and is_datetime64_dtype(type(fill_value)): + pytest.xfail("mix of lack of upcasting, resp. wrong missing value") # special case for box_dtype box_dtype = np.dtype(datetime64_dtype) if box_dtype == "dt_dtype" else box_dtype From ac2cb8713884f12ba14e057936376d0cba23daf8 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 10:52:40 -0500 Subject: [PATCH 3/7] trim xfail cases --- pandas/tests/dtypes/cast/test_promote.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index 1881ce295a4a2..6a4c10fd2eaa9 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -334,14 +334,10 @@ def test_maybe_promote_any_with_datetime64( if boxed: if is_datetime64_dtype(dtype): - if box_dtype == object: - pytest.xfail("falsely upcasts to object") - elif box_dtype is None and not is_datetime64_dtype(type(fill_value)): + if box_dtype is None and not is_datetime64_dtype(type(fill_value)): pytest.xfail("falsely upcasts to object") else: - if box_dtype == "dt_dtype": - pytest.xfail("mix of lack of upcasting, resp. wrong missing value") - elif box_dtype is None and is_datetime64_dtype(type(fill_value)): + if box_dtype is None and is_datetime64_dtype(type(fill_value)): pytest.xfail("mix of lack of upcasting, resp. wrong missing value") # special case for box_dtype From 7137dda9c5597d4021e8c3457b7abb296705fa41 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 11:21:54 -0500 Subject: [PATCH 4/7] cleanup --- pandas/tests/dtypes/cast/test_promote.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index 6a4c10fd2eaa9..66bf73758b000 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -332,12 +332,12 @@ def test_maybe_promote_any_with_datetime64( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if boxed: + if boxed and box_dtype is None: if is_datetime64_dtype(dtype): - if box_dtype is None and not is_datetime64_dtype(type(fill_value)): + if not is_datetime64_dtype(type(fill_value)): pytest.xfail("falsely upcasts to object") else: - if box_dtype is None and is_datetime64_dtype(type(fill_value)): + if is_datetime64_dtype(type(fill_value)): pytest.xfail("mix of lack of upcasting, resp. wrong missing value") # special case for box_dtype From 5ac4da58593421f27b1f68512dec57faf170b20e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 11:29:19 -0500 Subject: [PATCH 5/7] cleanup box xfails --- pandas/tests/dtypes/cast/test_promote.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index 66bf73758b000..bd1abc716c8e8 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -524,15 +524,15 @@ def test_maybe_promote_any_with_timedelta64( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if is_timedelta64_dtype(dtype): - if boxed and ( - box_dtype == object - or (box_dtype is None and not is_timedelta64_dtype(type(fill_value))) - ): - pytest.xfail("falsely upcasts to object") - else: - if boxed and box_dtype is None and is_timedelta64_dtype(type(fill_value)): - pytest.xfail("does not upcast correctly") + if boxed: + if is_timedelta64_dtype(dtype): + if box_dtype == object: + pytest.xfail("falsely upcasts to object") + elif box_dtype is None and not is_timedelta64_dtype(type(fill_value)): + pytest.xfail("falsely upcasts to object") + else: + if box_dtype is None and is_timedelta64_dtype(type(fill_value)): + pytest.xfail("does not upcast correctly") # special case for box_dtype box_dtype = np.dtype(timedelta64_dtype) if box_dtype == "td_dtype" else box_dtype From 7fe25bc56632ecb63b4e3e730551078ce51140cc Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 2 Oct 2019 11:31:05 -0500 Subject: [PATCH 6/7] clean up not-boxed xfails --- pandas/tests/dtypes/cast/test_promote.py | 28 +++++++++--------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index bd1abc716c8e8..0c30cfb3d5e71 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -678,24 +678,16 @@ def test_maybe_promote_any_numpy_dtype_with_na( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if ( - dtype == bytes - and not boxed - and fill_value is not None - and fill_value is not NaT - ): - pytest.xfail("does not upcast to object") - elif dtype == "uint64" and not boxed and fill_value == iNaT: - pytest.xfail("does not upcast correctly") - # below: opinionated that iNaT should be interpreted as missing value - elif ( - not boxed - and (is_float_dtype(dtype) or is_complex_dtype(dtype)) - and fill_value == iNaT - ): - pytest.xfail("does not cast to missing value marker correctly") - elif (is_string_dtype(dtype) or dtype == bool) and not boxed and fill_value == iNaT: - pytest.xfail("does not cast to missing value marker correctly") + if not boxed: + if dtype == bytes and fill_value is not None and fill_value is not NaT: + pytest.xfail("does not upcast to object") + elif dtype == "uint64" and fill_value == iNaT: + pytest.xfail("does not upcast correctly") + # below: opinionated that iNaT should be interpreted as missing value + elif (is_float_dtype(dtype) or is_complex_dtype(dtype)) and fill_value == iNaT: + pytest.xfail("does not cast to missing value marker correctly") + elif (is_string_dtype(dtype) or dtype == bool) and fill_value == iNaT: + pytest.xfail("does not cast to missing value marker correctly") if is_integer_dtype(dtype) and dtype == "uint64" and fill_value == iNaT: # uint64 + negative int casts to object; iNaT is considered as missing From 47413849803d2857fe5c0df4f53aa1f67eb9642d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 3 Oct 2019 14:08:09 -0500 Subject: [PATCH 7/7] remove xfails --- pandas/tests/dtypes/cast/test_promote.py | 65 ++++++++++++++---------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index b1390e07067a6..b3ff28e51ddc5 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -431,13 +431,18 @@ def test_maybe_promote_any_with_datetime64( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if boxed and box_dtype is None: - if is_datetime64_dtype(dtype): - if not is_datetime64_dtype(type(fill_value)): - pytest.xfail("falsely upcasts to object") - else: - if is_datetime64_dtype(type(fill_value)): - pytest.xfail("mix of lack of upcasting, resp. wrong missing value") + if is_datetime64_dtype(dtype): + if boxed and ( + box_dtype == object + or (box_dtype is None and not is_datetime64_dtype(type(fill_value))) + ): + pytest.xfail("falsely upcasts to object") + else: + if boxed and ( + box_dtype == "dt_dtype" + or (box_dtype is None and is_datetime64_dtype(type(fill_value))) + ): + pytest.xfail("mix of lack of upcasting, resp. wrong missing value") # special case for box_dtype box_dtype = np.dtype(datetime64_dtype) if box_dtype == "dt_dtype" else box_dtype @@ -659,15 +664,15 @@ def test_maybe_promote_any_with_timedelta64( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if boxed: - if is_timedelta64_dtype(dtype): - if box_dtype == object: - pytest.xfail("falsely upcasts to object") - elif box_dtype is None and not is_timedelta64_dtype(type(fill_value)): - pytest.xfail("falsely upcasts to object") - else: - if box_dtype is None and is_timedelta64_dtype(type(fill_value)): - pytest.xfail("does not upcast correctly") + if is_timedelta64_dtype(dtype): + if boxed and ( + box_dtype == object + or (box_dtype is None and not is_timedelta64_dtype(type(fill_value))) + ): + pytest.xfail("falsely upcasts to object") + else: + if boxed and box_dtype is None and is_timedelta64_dtype(type(fill_value)): + pytest.xfail("does not upcast correctly") # special case for box_dtype box_dtype = np.dtype(timedelta64_dtype) if box_dtype == "td_dtype" else box_dtype @@ -813,16 +818,24 @@ def test_maybe_promote_any_numpy_dtype_with_na( dtype = np.dtype(any_numpy_dtype_reduced) boxed, box_dtype = box # read from parametrized fixture - if not boxed: - if dtype == bytes and fill_value is not None and fill_value is not NaT: - pytest.xfail("does not upcast to object") - elif dtype == "uint64" and fill_value == iNaT: - pytest.xfail("does not upcast correctly") - # below: opinionated that iNaT should be interpreted as missing value - elif (is_float_dtype(dtype) or is_complex_dtype(dtype)) and fill_value == iNaT: - pytest.xfail("does not cast to missing value marker correctly") - elif (is_string_dtype(dtype) or dtype == bool) and fill_value == iNaT: - pytest.xfail("does not cast to missing value marker correctly") + if ( + dtype == bytes + and not boxed + and fill_value is not None + and fill_value is not NaT + ): + pytest.xfail("does not upcast to object") + elif dtype == "uint64" and not boxed and fill_value == iNaT: + pytest.xfail("does not upcast correctly") + # below: opinionated that iNaT should be interpreted as missing value + elif ( + not boxed + and (is_float_dtype(dtype) or is_complex_dtype(dtype)) + and fill_value == iNaT + ): + pytest.xfail("does not cast to missing value marker correctly") + elif (is_string_dtype(dtype) or dtype == bool) and not boxed and fill_value == iNaT: + pytest.xfail("does not cast to missing value marker correctly") if is_integer_dtype(dtype) and dtype == "uint64" and fill_value == iNaT: # uint64 + negative int casts to object; iNaT is considered as missing