Merge remote-tracking branch 'upstream/master' into truncate-decreasing

Author: dsaxton

asv_bench/benchmarks/arithmetic.py

@@ -67,7 +67,7 @@ def time_series_op_with_fill_value_no_nas(self):
         self.ser.add(self.ser, fill_value=4)
 
 
-class MixedFrameWithSeriesAxis0:
+class MixedFrameWithSeriesAxis:
     params = [
         [
             "eq",
@@ -78,7 +78,7 @@ class MixedFrameWithSeriesAxis0:
             "gt",
             "add",
             "sub",
-            "div",
+            "truediv",
             "floordiv",
             "mul",
             "pow",
@@ -87,15 +87,19 @@ class MixedFrameWithSeriesAxis0:
     param_names = ["opname"]
 
     def setup(self, opname):
-        arr = np.arange(10 ** 6).reshape(100, -1)
+        arr = np.arange(10 ** 6).reshape(1000, -1)
         df = DataFrame(arr)
         df["C"] = 1.0
         self.df = df
         self.ser = df[0]
+        self.row = df.iloc[0]
 
     def time_frame_op_with_series_axis0(self, opname):
         getattr(self.df, opname)(self.ser, axis=0)
 
+    def time_frame_op_with_series_axis1(self, opname):
+        getattr(operator, opname)(self.df, self.ser)
+
 
 class Ops:
 

asv_bench/benchmarks/stat_ops.py

@@ -11,8 +11,8 @@ class FrameOps:
     param_names = ["op", "dtype", "axis"]
 
     def setup(self, op, dtype, axis):
-        if op == "mad" and dtype == "Int64" and axis == 1:
-            # GH-33036
+        if op == "mad" and dtype == "Int64":
+            # GH-33036, GH#33600
             raise NotImplementedError
         values = np.random.randn(100000, 4)
         if dtype == "Int64":

doc/source/user_guide/computation.rst

@@ -318,8 +318,8 @@ We provide a number of common statistical functions:
     :meth:`~Rolling.kurt`, Sample kurtosis (4th moment)
     :meth:`~Rolling.quantile`, Sample quantile (value at %)
     :meth:`~Rolling.apply`, Generic apply
-    :meth:`~Rolling.cov`, Unbiased covariance (binary)
-    :meth:`~Rolling.corr`, Correlation (binary)
+    :meth:`~Rolling.cov`, Sample covariance (binary)
+    :meth:`~Rolling.corr`, Sample correlation (binary)
 
 .. _computation.window_variance.caveats:
 
@@ -341,6 +341,8 @@ We provide a number of common statistical functions:
    sample variance under the circumstances would result in a biased estimator
    of the variable we are trying to determine.
 
+   The same caveats apply to using any supported statistical sample methods.
+
 .. _stats.rolling_apply:
 
 Rolling apply
@@ -870,12 +872,12 @@ Method summary
     :meth:`~Expanding.max`, Maximum
     :meth:`~Expanding.std`, Sample standard deviation
     :meth:`~Expanding.var`, Sample variance
-    :meth:`~Expanding.skew`, Unbiased skewness (3rd moment)
-    :meth:`~Expanding.kurt`, Unbiased kurtosis (4th moment)
+    :meth:`~Expanding.skew`, Sample skewness (3rd moment)
+    :meth:`~Expanding.kurt`, Sample kurtosis (4th moment)
     :meth:`~Expanding.quantile`, Sample quantile (value at %)
     :meth:`~Expanding.apply`, Generic apply
-    :meth:`~Expanding.cov`, Unbiased covariance (binary)
-    :meth:`~Expanding.corr`, Correlation (binary)
+    :meth:`~Expanding.cov`, Sample covariance (binary)
+    :meth:`~Expanding.corr`, Sample correlation (binary)
 
 .. note::
 
@@ -884,6 +886,8 @@ Method summary
    windows. See :ref:`this section <computation.window_variance.caveats>` for more
    information.
 
+   The same caveats apply to using any supported statistical sample methods.
+
 .. currentmodule:: pandas
 
 Aside from not having a ``window`` parameter, these functions have the same

doc/source/whatsnew/v1.1.0.rst

@@ -175,8 +175,8 @@ Other API changes
 - Added :meth:`DataFrame.value_counts` (:issue:`5377`)
 - :meth:`Groupby.groups` now returns an abbreviated representation when called on large dataframes (:issue:`1135`)
 - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`)
-- Using a :func:`pandas.api.indexers.BaseIndexer` with ``skew``, ``cov``, ``corr`` will now raise a ``NotImplementedError`` (:issue:`32865`)
-- Using a :func:`pandas.api.indexers.BaseIndexer` with ``count``, ``min``, ``max``, ``median`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`)
+- Using a :func:`pandas.api.indexers.BaseIndexer` with ``cov``, ``corr`` will now raise a ``NotImplementedError`` (:issue:`32865`)
+- Using a :func:`pandas.api.indexers.BaseIndexer` with ``count``, ``min``, ``max``, ``median``, ``skew`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`)
 - Added a :func:`pandas.api.indexers.FixedForwardWindowIndexer` class to support forward-looking windows during ``rolling`` operations.
 -
 
@@ -191,6 +191,7 @@ Backwards incompatible API changes
   Previously a ``UnsupportedFunctionCall`` was raised (``AssertionError`` if ``min_count`` passed into :meth:`~DataFrameGroupby.median`) (:issue:`31485`)
 - :meth:`DataFrame.at` and :meth:`Series.at` will raise a ``TypeError`` instead of a ``ValueError`` if an incompatible key is passed, and ``KeyError`` if a missing key is passed, matching the behavior of ``.loc[]`` (:issue:`31722`)
 - Passing an integer dtype other than ``int64`` to ``np.array(period_index, dtype=...)`` will now raise ``TypeError`` instead of incorrectly using ``int64`` (:issue:`32255`)
+- Passing an invalid ``fill_value`` to :meth:`Categorical.take` raises a ``ValueError`` instead of ``TypeError`` (:issue:`33660`)
 
 ``MultiIndex.get_indexer`` interprets `method` argument differently
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -324,6 +325,36 @@ key and type of :class:`Index`.  These now consistently raise ``KeyError`` (:iss
     ...
     KeyError: Timestamp('1970-01-01 00:00:00')
 
+.. _whatsnew_110.api_breaking.indexing_int_multiindex_raises_key_errors:
+
+Failed Integer Lookups on MultiIndex Raise KeyError
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Indexing with integers with a :class:`MultiIndex` that has a integer-dtype
+first level incorrectly failed to raise ``KeyError`` when one or more of
+those integer keys is not present in the first level of the index (:issue:`33539`)
+
+.. ipython:: python
+
+    idx = pd.Index(range(4))
+    dti = pd.date_range("2000-01-03", periods=3)
+    mi = pd.MultiIndex.from_product([idx, dti])
+    ser = pd.Series(range(len(mi)), index=mi)
+
+*Previous behavior*:
+
+.. code-block:: ipython
+
+    In [5]: ser[[5]]
+    Out[5]: Series([], dtype: int64)
+
+*New behavior*:
+
+.. code-block:: ipython
+
+    In [5]: ser[[5]]
+    ...
+    KeyError: '[5] not in index'
+
 :meth:`DataFrame.merge` preserves right frame's row order
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 :meth:`DataFrame.merge` now preserves right frame's row order when executing a right merge (:issue:`27453`)
@@ -419,6 +450,7 @@ Performance improvements
 - Performance improvement in :class:`Timedelta` constructor (:issue:`30543`)
 - Performance improvement in :class:`Timestamp` constructor (:issue:`30543`)
 - Performance improvement in flex arithmetic ops between :class:`DataFrame` and :class:`Series` with ``axis=0`` (:issue:`31296`)
+- Performance improvement in  arithmetic ops between :class:`DataFrame` and :class:`Series` with ``axis=1`` (:issue:`33600`)
 - The internal index method :meth:`~Index._shallow_copy` now copies cached attributes over to the new index,
   avoiding creating these again on the new index. This can speed up many operations that depend on creating copies of
   existing indexes (:issue:`28584`, :issue:`32640`, :issue:`32669`)
@@ -462,6 +494,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.to_period` not infering the frequency when called with no arguments (:issue:`33358`)
 - Bug in :meth:`DatetimeIndex.tz_localize` incorrectly retaining ``freq`` in some cases where the original freq is no longer valid (:issue:`30511`)
 - Bug in :meth:`DatetimeIndex.intersection` losing ``freq`` and timezone in some cases (:issue:`33604`)
+- Bug in :class:`DatetimeIndex` addition and subtraction with some types of :class:`DateOffset` objects incorrectly retaining an invalid ``freq`` attribute (:issue:`33779`)
 
 Timedelta
 ^^^^^^^^^
@@ -499,6 +532,7 @@ Strings
 ^^^^^^^
 
 - Bug in the :meth:`~Series.astype` method when converting "string" dtype data to nullable integer dtype (:issue:`32450`).
+- Fixed issue where taking ``min`` or ``max`` of a ``StringArray`` or ``Series`` with ``StringDtype`` type would raise. (:issue:`31746`)
 - Bug in :meth:`Series.str.cat` returning ``NaN`` output when other had :class:`Index` type (:issue:`33425`)
 
 
@@ -661,6 +695,7 @@ Other
 - Bug in :meth:`Series.map` not raising on invalid ``na_action`` (:issue:`32815`)
 - Bug in :meth:`DataFrame.__dir__` caused a segfault when using unicode surrogates in a column name (:issue:`25509`)
 - Bug in :meth:`DataFrame.plot.scatter` caused an error when plotting variable marker sizes (:issue:`32904`)
+- :class:`IntegerArray` now implements the ``sum`` operation (:issue:`33172`)
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/hashtable_func_helper.pxi.in

@@ -125,7 +125,7 @@ cpdef value_count_{{dtype}}({{c_type}}[:] values, bint dropna):
 {{if dtype == 'object'}}
 def duplicated_{{dtype}}(ndarray[{{dtype}}] values, object keep='first'):
 {{else}}
-def duplicated_{{dtype}}({{c_type}}[:] values, object keep='first'):
+def duplicated_{{dtype}}(const {{c_type}}[:] values, object keep='first'):
 {{endif}}
     cdef:
         int ret = 0

pandas/_libs/index.pyx

@@ -21,14 +21,13 @@ cnp.import_array()
 
 cimport pandas._libs.util as util
 
-from pandas._libs.tslibs import Period
+from pandas._libs.tslibs import Period, Timedelta
 from pandas._libs.tslibs.nattype cimport c_NaT as NaT
 from pandas._libs.tslibs.c_timestamp cimport _Timestamp
 
 from pandas._libs.hashtable cimport HashTable
 
 from pandas._libs import algos, hashtable as _hash
-from pandas._libs.tslibs import Timedelta, period as periodlib
 from pandas._libs.missing import checknull
 
 
@@ -441,6 +440,10 @@ cdef class DatetimeEngine(Int64Engine):
         except KeyError:
             raise KeyError(val)
 
+    def get_indexer_non_unique(self, targets):
+        # we may get datetime64[ns] or timedelta64[ns], cast these to int64
+        return super().get_indexer_non_unique(targets.view("i8"))
+
     def get_indexer(self, values):
         self._ensure_mapping_populated()
         if values.dtype != self._get_box_dtype():
@@ -501,38 +504,6 @@ cdef class PeriodEngine(Int64Engine):
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=True)
 
-    def get_indexer(self, values):
-        cdef:
-            ndarray[int64_t, ndim=1] ordinals
-
-        super(PeriodEngine, self)._ensure_mapping_populated()
-
-        freq = super(PeriodEngine, self).vgetter().freq
-        ordinals = periodlib.extract_ordinals(values, freq)
-
-        return self.mapping.lookup(ordinals)
-
-    def get_pad_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
-        freq = super(PeriodEngine, self).vgetter().freq
-        ordinal = periodlib.extract_ordinals(other, freq)
-
-        return algos.pad(self._get_index_values(),
-                         np.asarray(ordinal), limit=limit)
-
-    def get_backfill_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
-        freq = super(PeriodEngine, self).vgetter().freq
-        ordinal = periodlib.extract_ordinals(other, freq)
-
-        return algos.backfill(self._get_index_values(),
-                              np.asarray(ordinal), limit=limit)
-
-    def get_indexer_non_unique(self, targets):
-        freq = super(PeriodEngine, self).vgetter().freq
-        ordinal = periodlib.extract_ordinals(targets, freq)
-        ordinal_array = np.asarray(ordinal)
-
-        return super(PeriodEngine, self).get_indexer_non_unique(ordinal_array)
-
 
 cdef class BaseMultiIndexCodesEngine:
     """

pandas/_libs/lib.pyx

@@ -1173,15 +1173,15 @@ cdef class Seen:
                     or self.nat_)
 
 
-cdef object _try_infer_map(object v):
+cdef object _try_infer_map(object dtype):
     """
     If its in our map, just return the dtype.
     """
     cdef:
         object val
         str attr
-    for attr in ['name', 'kind', 'base']:
-        val = getattr(v.dtype, attr)
+    for attr in ["name", "kind", "base"]:
+        val = getattr(dtype, attr)
         if val in _TYPE_MAP:
             return _TYPE_MAP[val]
     return None
@@ -1294,44 +1294,49 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
 
     if util.is_array(value):
         values = value
-    elif hasattr(value, 'dtype'):
+    elif hasattr(value, "inferred_type") and skipna is False:
+        # Index, use the cached attribute if possible, populate the cache otherwise
+        return value.inferred_type
+    elif hasattr(value, "dtype"):
         # this will handle ndarray-like
         # e.g. categoricals
-        try:
-            values = getattr(value, '_values', getattr(value, 'values', value))
-        except TypeError:
-            # This gets hit if we have an EA, since cython expects `values`
-            #  to be an ndarray
-            value = _try_infer_map(value)
+        dtype = value.dtype
+        if not isinstance(dtype, np.dtype):
+            value = _try_infer_map(value.dtype)
             if value is not None:
                 return value
 
-            # its ndarray like but we can't handle
+            # its ndarray-like but we can't handle
             raise ValueError(f"cannot infer type for {type(value)}")
 
+        # Unwrap Series/Index
+        values = np.asarray(value)
+
     else:
         if not isinstance(value, list):
             value = list(value)
+
         from pandas.core.dtypes.cast import (
             construct_1d_object_array_from_listlike)
         values = construct_1d_object_array_from_listlike(value)
 
     # make contiguous
-    values = values.ravel()
+    # for f-contiguous array 1000 x 1000, passing order="K" gives 5000x speedup
+    values = values.ravel(order="K")
 
-    val = _try_infer_map(values)
+    val = _try_infer_map(values.dtype)
     if val is not None:
         return val
 
     if values.dtype != np.object_:
-        values = values.astype('O')
+        values = values.astype("O")
 
     if skipna:
         values = values[~isnaobj(values)]
 
     n = len(values)
     if n == 0:
-        return 'empty'
+        return "empty"
 
     # try to use a valid value
     for i in range(n):

pandas/compat/numpy/function.py

@@ -218,7 +218,7 @@ def validate_cum_func_with_skipna(skipna, args, kwargs, name):
 LOGICAL_FUNC_DEFAULTS = dict(out=None, keepdims=False)
 validate_logical_func = CompatValidator(LOGICAL_FUNC_DEFAULTS, method="kwargs")
 
-MINMAX_DEFAULTS = dict(out=None, keepdims=False)
+MINMAX_DEFAULTS = dict(axis=None, out=None, keepdims=False)
 validate_min = CompatValidator(
     MINMAX_DEFAULTS, fname="min", method="both", max_fname_arg_count=1
 )
@@ -251,10 +251,16 @@ def validate_cum_func_with_skipna(skipna, args, kwargs, name):
 STAT_FUNC_DEFAULTS["dtype"] = None
 STAT_FUNC_DEFAULTS["out"] = None
 
-PROD_DEFAULTS = SUM_DEFAULTS = STAT_FUNC_DEFAULTS.copy()
+SUM_DEFAULTS = STAT_FUNC_DEFAULTS.copy()
+SUM_DEFAULTS["axis"] = None
 SUM_DEFAULTS["keepdims"] = False
 SUM_DEFAULTS["initial"] = None
 
+PROD_DEFAULTS = STAT_FUNC_DEFAULTS.copy()
+PROD_DEFAULTS["axis"] = None
+PROD_DEFAULTS["keepdims"] = False
+PROD_DEFAULTS["initial"] = None
+
 MEDIAN_DEFAULTS = STAT_FUNC_DEFAULTS.copy()
 MEDIAN_DEFAULTS["overwrite_input"] = False
 MEDIAN_DEFAULTS["keepdims"] = False

pandas/core/algorithms.py

@@ -49,6 +49,7 @@
     ABCExtensionArray,
     ABCIndex,
     ABCIndexClass,
+    ABCMultiIndex,
     ABCSeries,
 )
 from pandas.core.dtypes.missing import isna, na_value_for_dtype
@@ -89,6 +90,10 @@ def _ensure_data(values, dtype=None):
     values : ndarray
     pandas_dtype : str or dtype
     """
+    if not isinstance(values, ABCMultiIndex):
+        # extract_array would raise
+        values = extract_array(values, extract_numpy=True)
+
     # we check some simple dtypes first
     if is_object_dtype(dtype):
         return ensure_object(np.asarray(values)), "object"
@@ -151,7 +156,6 @@ def _ensure_data(values, dtype=None):
     elif is_categorical_dtype(values) and (
         is_categorical_dtype(dtype) or dtype is None
     ):
-        values = getattr(values, "values", values)
         values = values.codes
         dtype = "category"