Merge remote-tracking branch 'upstream/master' into numba

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -153,13 +153,17 @@ jobs:
       run: |
         source activate pandas-dev
         pytest pandas/tests/frame/methods --array-manager
+        pytest pandas/tests/frame/test_reductions.py --array-manager
+        pytest pandas/tests/reductions/ --array-manager
+        pytest pandas/tests/generic/test_generic.py --array-manager
         pytest pandas/tests/arithmetic/ --array-manager
+        pytest pandas/tests/groupby/aggregate/ --array-manager
         pytest pandas/tests/reshape/merge --array-manager
 
         # indexing subset (temporary since other tests don't pass yet)
         pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean --array-manager
         pytest pandas/tests/frame/indexing/test_where.py --array-manager
-        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_multi_index --array-manager
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index --array-manager
         pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns --array-manager
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups --array-manager
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column --array-manager

doc/source/whatsnew/v1.2.3.rst

@@ -16,8 +16,10 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
 - Fixed regression in :meth:`~DataFrame.to_excel` raising ``KeyError`` when giving duplicate columns with ``columns`` attribute (:issue:`39695`)
-- Fixed regression in :class:`IntegerArray` unary ops propagating mask on assignment (:issue:`39943`)
+- Fixed regression in nullable integer unary ops propagating mask on assignment (:issue:`39943`)
 - Fixed regression in :meth:`DataFrame.__setitem__` not aligning :class:`DataFrame` on right-hand side for boolean indexer (:issue:`39931`)
+- Fixed regression in :meth:`~DataFrame.to_json` failing to use ``compression`` with URL-like paths that are internally opened in binary mode or with user-provided file objects that are opened in binary mode (:issue:`39985`)
+-
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.3.0.rst

@@ -308,6 +308,7 @@ Timedelta
 - Bug in constructing :class:`Timedelta` from ``np.timedelta64`` objects with non-nanosecond units that are out of bounds for ``timedelta64[ns]`` (:issue:`38965`)
 - Bug in constructing a :class:`TimedeltaIndex` incorrectly accepting ``np.datetime64("NaT")`` objects (:issue:`39462`)
 - Bug in constructing :class:`Timedelta` from input string with only symbols and no digits failed to raise an error (:issue:`39710`)
+- Bug in :class:`TimedeltaIndex` and :func:`to_timedelta` failing to raise when passed non-nanosecond ``timedelta64`` arrays that overflow when converting to ``timedelta64[ns]`` (:issue:`40008`)
 
 Timezones
 ^^^^^^^^^
@@ -323,6 +324,8 @@ Numeric
 - Bug in :meth:`DataFrame.rank` with ``np.inf`` and mixture of ``np.nan`` and ``np.inf`` (:issue:`32593`)
 - Bug in :meth:`DataFrame.rank` with ``axis=0`` and columns holding incomparable types raising ``IndexError`` (:issue:`38932`)
 - Bug in :func:`select_dtypes` different behavior between Windows and Linux with ``include="int"`` (:issue:`36569`)
+- Bug in :meth:`DataFrame.apply` and :meth:`DataFrame.agg` when passed argument ``func="size"`` would operate on the entire ``DataFrame`` instead of rows or columns (:issue:`39934`)
+- Bug in :meth:`DataFrame.transform` would raise ``SpecificationError`` when passed a dictionary and columns were missing; will now raise a ``KeyError`` instead (:issue:`40004`)
 -
 
 Conversion
@@ -376,7 +379,7 @@ Missing
 ^^^^^^^
 
 - Bug in :class:`Grouper` now correctly propagates ``dropna`` argument and :meth:`DataFrameGroupBy.transform` now correctly handles missing values for ``dropna=True`` (:issue:`35612`)
--
+- Bug in :func:`isna`, and :meth:`Series.isna`, :meth:`Index.isna`, :meth:`DataFrame.isna` (and the corresponding ``notna`` functions) not recognizing ``Decimal("NaN")`` objects (:issue:`39409`)
 -
 
 MultiIndex
@@ -439,6 +442,10 @@ Groupby/resample/rolling
 - Bug in :meth:`core.window.expanding.ExpandingGroupby.corr` and :meth:`core.window.expanding.ExpandingGroupby.cov` where 1 would be returned instead of ``np.nan`` when providing ``other`` that was longer than each group (:issue:`39591`)
 - Bug in :meth:`.GroupBy.mean`, :meth:`.GroupBy.median` and :meth:`DataFrame.pivot_table` not propagating metadata (:issue:`28283`)
 - Bug in :meth:`Series.rolling` and :meth:`DataFrame.rolling` not calculating window bounds correctly when window is an offset and dates are in descending order (:issue:`40002`)
+- Bug in :class:`SeriesGroupBy` and :class:`DataFrameGroupBy` on an empty ``Series`` or ``DataFrame`` would lose index, columns, and/or data types when directly using the methods ``idxmax``, ``idxmin``, ``mad``, ``min``, ``max``, ``sum``, ``prod``, and ``skew`` or using them through ``apply``, ``aggregate``, or ``resample`` (:issue:`26411`)
+- Bug in :meth:`DataFrameGroupBy.apply` where a :class:`MultiIndex` would be created instead of an :class:`Index` if a :class:`:meth:`core.window.rolling.RollingGroupby` object was created (:issue:`39732`)
+- Bug in :meth:`DataFrameGroupBy.sample` where error was raised when ``weights`` was specified and the index was an :class:`Int64Index` (:issue:`39927`)
+- Bug in :meth:`DataFrameGroupBy.aggregate` and :meth:`.Resampler.aggregate` would sometimes raise ``SpecificationError`` when passed a dictionary and columns were missing; will now always raise a ``KeyError`` instead (:issue:`40004`)
 -
 
 Reshaping
@@ -454,6 +461,7 @@ Reshaping
 - Bug in :meth:`DataFrame.sort_values` not reshaping index correctly after sorting on columns, when ``ignore_index=True`` (:issue:`39464`)
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``ExtensionDtype`` dtypes (:issue:`39454`)
 - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``datetime64`` and ``timedelta64`` dtypes (:issue:`39574`)
+- Bug in :meth:`DataFrame.pivot_table` returning a ``MultiIndex`` for a single value when operating on and empty ``DataFrame`` (:issue:`13483`)
 
 Sparse
 ^^^^^^

pandas/_libs/missing.pyx

@@ -1,3 +1,4 @@
+from decimal import Decimal
 import numbers
 
 import cython
@@ -36,6 +37,8 @@ cdef:
 
     bint is_32bit = not IS64
 
+    type cDecimal = Decimal  # for faster isinstance checks
+
 
 cpdef bint is_matching_na(object left, object right, bint nan_matches_none=False):
     """
@@ -86,6 +89,8 @@ cpdef bint is_matching_na(object left, object right, bint nan_matches_none=False
             and util.is_timedelta64_object(right)
             and get_timedelta64_value(right) == NPY_NAT
         )
+    elif is_decimal_na(left):
+        return is_decimal_na(right)
     return False
 
 
@@ -113,7 +118,18 @@ cpdef bint checknull(object val):
     The difference between `checknull` and `checknull_old` is that `checknull`
     does *not* consider INF or NEGINF to be NA.
     """
-    return val is C_NA or is_null_datetimelike(val, inat_is_null=False)
+    return (
+        val is C_NA
+        or is_null_datetimelike(val, inat_is_null=False)
+        or is_decimal_na(val)
+    )
+
+
+cdef inline bint is_decimal_na(object val):
+    """
+    Is this a decimal.Decimal object Decimal("NAN").
+    """
+    return isinstance(val, cDecimal) and val != val
 
 
 cpdef bint checknull_old(object val):

pandas/_libs/tslibs/conversion.pyx

@@ -239,6 +239,11 @@ def ensure_datetime64ns(arr: ndarray, copy: bool=True):
         return result
 
     unit = get_datetime64_unit(arr.flat[0])
+    if unit == NPY_DATETIMEUNIT.NPY_FR_GENERIC:
+        # without raising explicitly here, we end up with a SystemError
+        # built-in function ensure_datetime64ns returned a result with an error
+        raise ValueError("datetime64/timedelta64 must have a unit specified")
+
     if unit == NPY_FR_ns:
         if copy:
             arr = arr.copy()
@@ -289,9 +294,8 @@ def ensure_timedelta64ns(arr: ndarray, copy: bool=True):
         else:
             bad_val = tdmax
 
-        raise OutOfBoundsTimedelta(
-            f"Out of bounds for nanosecond {arr.dtype.name} {bad_val}"
-        )
+        msg = f"Out of bounds for nanosecond {arr.dtype.name} {str(bad_val)}"
+        raise OutOfBoundsTimedelta(msg)
 
     return dt64_result.view(TD64NS_DTYPE)
 

pandas/_libs/tslibs/timedeltas.pyx

@@ -347,9 +347,13 @@ def array_to_timedelta64(ndarray[object] values, str unit=None, str errors="rais
         for i in range(n):
             try:
                 result[i] = convert_to_timedelta64(values[i], parsed_unit)
-            except ValueError:
+            except ValueError as err:
                 if errors == 'coerce':
                     result[i] = NPY_NAT
+                elif "unit abbreviation w/o a number" in str(err):
+                    # re-raise with more pertinent message
+                    msg = f"Could not convert '{values[i]}' to NumPy timedelta"
+                    raise ValueError(msg) from err
                 else:
                     raise
 

pandas/_testing/__init__.py

@@ -2,6 +2,7 @@
 
 import collections
 from datetime import datetime
+from decimal import Decimal
 from functools import wraps
 import operator
 import os
@@ -146,7 +147,7 @@
     + BYTES_DTYPES
 )
 
-NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA]
+NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA, Decimal("NaN")]
 
 EMPTY_STRING_PATTERN = re.compile("^$")
 

pandas/_testing/asserters.py

@@ -7,6 +7,7 @@
 import numpy as np
 
 from pandas._libs.lib import no_default
+from pandas._libs.missing import is_matching_na
 import pandas._libs.testing as _testing
 
 from pandas.core.dtypes.common import (
@@ -18,6 +19,7 @@
     is_numeric_dtype,
     needs_i8_conversion,
 )
+from pandas.core.dtypes.dtypes import PandasDtype
 from pandas.core.dtypes.missing import array_equivalent
 
 import pandas as pd
@@ -457,22 +459,8 @@ def assert_attr_equal(attr: str, left, right, obj: str = "Attributes"):
 
     if left_attr is right_attr:
         return True
-    elif (
-        is_number(left_attr)
-        and np.isnan(left_attr)
-        and is_number(right_attr)
-        and np.isnan(right_attr)
-    ):
-        # np.nan
-        return True
-    elif (
-        isinstance(left_attr, (np.datetime64, np.timedelta64))
-        and isinstance(right_attr, (np.datetime64, np.timedelta64))
-        and type(left_attr) is type(right_attr)
-        and np.isnat(left_attr)
-        and np.isnat(right_attr)
-    ):
-        # np.datetime64("nat") or np.timedelta64("nat")
+    elif is_matching_na(left_attr, right_attr):
+        # e.g. both np.nan, both NaT, both pd.NA, ...
         return True
 
     try:
@@ -630,12 +618,12 @@ def raise_assert_detail(obj, message, left, right, diff=None, index_values=None)
 
     if isinstance(left, np.ndarray):
         left = pprint_thing(left)
-    elif is_categorical_dtype(left):
+    elif is_categorical_dtype(left) or isinstance(left, PandasDtype):
         left = repr(left)
 
     if isinstance(right, np.ndarray):
         right = pprint_thing(right)
-    elif is_categorical_dtype(right):
+    elif is_categorical_dtype(right) or isinstance(right, PandasDtype):
         right = repr(right)
 
     msg += f"""

pandas/conftest.py

@@ -304,7 +304,7 @@ def nselect_method(request):
 # ----------------------------------------------------------------
 # Missing values & co.
 # ----------------------------------------------------------------
-@pytest.fixture(params=tm.NULL_OBJECTS, ids=str)
+@pytest.fixture(params=tm.NULL_OBJECTS, ids=lambda x: type(x).__name__)
 def nulls_fixture(request):
     """
     Fixture for each null type in pandas.