Merge remote-tracking branch 'upstream/master' into corr-with-na

Author: dsaxton

doc/source/whatsnew/v1.1.0.rst

@@ -98,6 +98,8 @@ Other enhancements
   This can be used to set a custom compression level, e.g.,
   ``df.to_csv(path, compression={'method': 'gzip', 'compresslevel': 1}``
   (:issue:`33196`)
+- :meth:`Series.update` now accepts objects that can be coerced to a :class:`Series`,
+  such as ``dict`` and ``list``, mirroring the behavior of :meth:`DataFrame.update` (:issue:`33215`)
 - :meth:`~pandas.core.groupby.GroupBy.transform` and :meth:`~pandas.core.groupby.GroupBy.aggregate` has gained ``engine`` and ``engine_kwargs`` arguments that supports executing functions with ``Numba`` (:issue:`32854`, :issue:`33388`)
 - :meth:`~pandas.core.resample.Resampler.interpolate` now supports SciPy interpolation method :class:`scipy.interpolate.CubicSpline` as method ``cubicspline`` (:issue:`33670`)
 -
@@ -566,12 +568,14 @@ Indexing
 - Bug in :meth:`DatetimeIndex.insert` and :meth:`TimedeltaIndex.insert` causing index ``freq`` to be lost when setting an element into an empty :class:`Series` (:issue:33573`)
 - Bug in :meth:`Series.__setitem__` with an :class:`IntervalIndex` and a list-like key of integers (:issue:`33473`)
 - Bug in :meth:`Series.__getitem__` allowing missing labels with ``np.ndarray``, :class:`Index`, :class:`Series` indexers but not ``list``, these now all raise ``KeyError`` (:issue:`33646`)
+- Bug in :meth:`DataFrame.truncate` and :meth:`Series.truncate` where index was assumed to be monotone increasing (:issue:`33756`)
 
 Missing
 ^^^^^^^
 - Calling :meth:`fillna` on an empty Series now correctly returns a shallow copied object. The behaviour is now consistent with :class:`Index`, :class:`DataFrame` and a non-empty :class:`Series` (:issue:`32543`).
 - Bug in :meth:`replace` when argument ``to_replace`` is of type dict/list and is used on a :class:`Series` containing ``<NA>`` was raising a ``TypeError``. The method now handles this by ignoring ``<NA>`` values when doing the comparison for the replacement (:issue:`32621`)
 - Bug in :meth:`~Series.any` and :meth:`~Series.all` incorrectly returning ``<NA>`` for all ``False`` or all ``True`` values using the nulllable boolean dtype and with ``skipna=False`` (:issue:`33253`)
+- Clarified documentation on interpolate with method =akima. The ``der`` parameter must be scalar or None (:issue:`33426`)
 
 MultiIndex
 ^^^^^^^^^^
@@ -621,6 +625,9 @@ I/O
   unsupported HDF file (:issue:`9539`)
 - Bug in :meth:`~DataFrame.to_parquet` was not raising ``PermissionError`` when writing to a private s3 bucket with invalid creds. (:issue:`27679`)
 - Bug in :meth:`~DataFrame.to_csv` was silently failing when writing to an invalid s3 bucket. (:issue:`32486`)
+- Bug in :meth:`~DataFrame.read_feather` was raising an `ArrowIOError` when reading an s3 or http file path (:issue:`29055`)
+- Bug in :meth:`read_parquet` was raising a ``FileNotFoundError`` when passed an s3 directory path. (:issue:`26388`)
+- Bug in :meth:`~DataFrame.to_parquet` was throwing an ``AttributeError`` when writing a partitioned parquet file to s3 (:issue:`27596`)
 
 Plotting
 ^^^^^^^^

pandas/_testing.py

@@ -1024,7 +1024,12 @@ def _raise(left, right, err_msg):
 
 
 def assert_extension_array_equal(
-    left, right, check_dtype=True, check_less_precise=False, check_exact=False
+    left,
+    right,
+    check_dtype=True,
+    check_less_precise=False,
+    check_exact=False,
+    index_values=None,
 ):
     """
     Check that left and right ExtensionArrays are equal.
@@ -1041,6 +1046,8 @@ def assert_extension_array_equal(
         If int, then specify the digits to compare.
     check_exact : bool, default False
         Whether to compare number exactly.
+    index_values : numpy.ndarray, default None
+        optional index (shared by both left and right), used in output.
 
     Notes
     -----
@@ -1056,24 +1063,31 @@ def assert_extension_array_equal(
     if hasattr(left, "asi8") and type(right) == type(left):
         # Avoid slow object-dtype comparisons
         # np.asarray for case where we have a np.MaskedArray
-        assert_numpy_array_equal(np.asarray(left.asi8), np.asarray(right.asi8))
+        assert_numpy_array_equal(
+            np.asarray(left.asi8), np.asarray(right.asi8), index_values=index_values
+        )
         return
 
     left_na = np.asarray(left.isna())
     right_na = np.asarray(right.isna())
-    assert_numpy_array_equal(left_na, right_na, obj="ExtensionArray NA mask")
+    assert_numpy_array_equal(
+        left_na, right_na, obj="ExtensionArray NA mask", index_values=index_values
+    )
 
     left_valid = np.asarray(left[~left_na].astype(object))
     right_valid = np.asarray(right[~right_na].astype(object))
     if check_exact:
-        assert_numpy_array_equal(left_valid, right_valid, obj="ExtensionArray")
+        assert_numpy_array_equal(
+            left_valid, right_valid, obj="ExtensionArray", index_values=index_values
+        )
     else:
         _testing.assert_almost_equal(
             left_valid,
             right_valid,
             check_dtype=check_dtype,
             check_less_precise=check_less_precise,
             obj="ExtensionArray",
+            index_values=index_values,
         )
 
 
@@ -1206,12 +1220,17 @@ def assert_series_equal(
             check_less_precise=check_less_precise,
             check_dtype=check_dtype,
             obj=str(obj),
+            index_values=np.asarray(left.index),
         )
     elif is_extension_array_dtype(left.dtype) and is_extension_array_dtype(right.dtype):
-        assert_extension_array_equal(left._values, right._values)
+        assert_extension_array_equal(
+            left._values, right._values, index_values=np.asarray(left.index)
+        )
     elif needs_i8_conversion(left.dtype) or needs_i8_conversion(right.dtype):
         # DatetimeArray or TimedeltaArray
-        assert_extension_array_equal(left._values, right._values)
+        assert_extension_array_equal(
+            left._values, right._values, index_values=np.asarray(left.index)
+        )
     else:
         _testing.assert_almost_equal(
             left._values,
@@ -1429,6 +1448,8 @@ def assert_equal(left, right, **kwargs):
 
     if isinstance(left, pd.Index):
         assert_index_equal(left, right, **kwargs)
+        if isinstance(left, (pd.DatetimeIndex, pd.TimedeltaIndex)):
+            assert left.freq == right.freq, (left.freq, right.freq)
     elif isinstance(left, pd.Series):
         assert_series_equal(left, right, **kwargs)
     elif isinstance(left, pd.DataFrame):

pandas/core/arrays/categorical.py

@@ -463,7 +463,7 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
                 return self
             return self._set_dtype(dtype)
         if is_extension_array_dtype(dtype):
-            return array(self, dtype=dtype, copy=copy)  # type: ignore # GH 28770
+            return array(self, dtype=dtype, copy=copy)
         if is_integer_dtype(dtype) and self.isna().any():
             raise ValueError("Cannot convert float NaN to integer")
         return np.array(self, dtype=dtype, copy=copy)

pandas/core/arrays/datetimelike.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
 import operator
-from typing import Any, Sequence, Type, Union, cast
+from typing import Any, Sequence, Type, TypeVar, Union, cast
 import warnings
 
 import numpy as np
@@ -410,15 +410,15 @@ def ceil(self, freq, ambiguous="raise", nonexistent="raise"):
 
     def _with_freq(self, freq):
         """
-        Helper to set our freq in-place, returning self to allow method chaining.
+        Helper to get a view on the same data, with a new freq.
 
         Parameters
         ----------
         freq : DateOffset, None, or "infer"
 
         Returns
         -------
-        self
+        Same type as self
         """
         # GH#29843
         if freq is None:
@@ -433,8 +433,12 @@ def _with_freq(self, freq):
             assert freq == "infer"
             freq = frequencies.to_offset(self.inferred_freq)
 
-        self._freq = freq
-        return self
+        arr = self.view()
+        arr._freq = freq
+        return arr
+
+
+DatetimeLikeArrayT = TypeVar("DatetimeLikeArrayT", bound="DatetimeLikeArrayMixin")
 
 
 class DatetimeLikeArrayMixin(
@@ -679,7 +683,7 @@ def _concat_same_type(cls, to_concat, axis: int = 0):
 
         return cls._simple_new(values, dtype=dtype, freq=new_freq)
 
-    def copy(self):
+    def copy(self: DatetimeLikeArrayT) -> DatetimeLikeArrayT:
         values = self.asi8.copy()
         return type(self)._simple_new(values, dtype=self.dtype, freq=self.freq)
 

pandas/core/arrays/period.py

@@ -1,6 +1,6 @@
 from datetime import timedelta
 import operator
-from typing import Any, Callable, List, Optional, Sequence, Union
+from typing import Any, Callable, List, Optional, Sequence, Type, Union
 
 import numpy as np
 
@@ -20,6 +20,7 @@
     period_asfreq_arr,
 )
 from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds
+from pandas._typing import AnyArrayLike
 from pandas.util._decorators import cache_readonly
 
 from pandas.core.dtypes.common import (
@@ -172,8 +173,8 @@ def _simple_new(cls, values: np.ndarray, freq=None, **kwargs) -> "PeriodArray":
 
     @classmethod
     def _from_sequence(
-        cls,
-        scalars: Sequence[Optional[Period]],
+        cls: Type["PeriodArray"],
+        scalars: Union[Sequence[Optional[Period]], AnyArrayLike],
         dtype: Optional[PeriodDtype] = None,
         copy: bool = False,
     ) -> "PeriodArray":
@@ -186,7 +187,6 @@ def _from_sequence(
             validate_dtype_freq(scalars.dtype, freq)
             if copy:
                 scalars = scalars.copy()
-            assert isinstance(scalars, PeriodArray)  # for mypy
             return scalars
 
         periods = np.asarray(scalars, dtype=object)
@@ -772,7 +772,7 @@ def raise_on_incompatible(left, right):
 
 
 def period_array(
-    data: Sequence[Optional[Period]],
+    data: Union[Sequence[Optional[Period]], AnyArrayLike],
     freq: Optional[Union[str, Tick]] = None,
     copy: bool = False,
 ) -> PeriodArray:

pandas/core/arrays/timedeltas.py

@@ -27,19 +27,15 @@
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
-from pandas.core.dtypes.generic import (
-    ABCDataFrame,
-    ABCIndexClass,
-    ABCSeries,
-    ABCTimedeltaIndex,
-)
+from pandas.core.dtypes.generic import ABCSeries, ABCTimedeltaIndex
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import nanops
 from pandas.core.algorithms import checked_add_with_arr
 from pandas.core.arrays import datetimelike as dtl
 import pandas.core.common as com
 from pandas.core.construction import extract_array
+from pandas.core.ops.common import unpack_zerodim_and_defer
 
 from pandas.tseries.frequencies import to_offset
 from pandas.tseries.offsets import Tick
@@ -456,12 +452,8 @@ def _addsub_object_array(self, other, op):
                 f"Cannot add/subtract non-tick DateOffset to {type(self).__name__}"
             ) from err
 
+    @unpack_zerodim_and_defer("__mul__")
     def __mul__(self, other):
-        other = lib.item_from_zerodim(other)
-
-        if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
-            return NotImplemented
-
         if is_scalar(other):
             # numpy will accept float and int, raise TypeError for others
             result = self._data * other
@@ -492,12 +484,9 @@ def __mul__(self, other):
 
     __rmul__ = __mul__
 
+    @unpack_zerodim_and_defer("__truediv__")
     def __truediv__(self, other):
         # timedelta / X is well-defined for timedelta-like or numeric X
-        other = lib.item_from_zerodim(other)
-
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
 
         if isinstance(other, (timedelta, np.timedelta64, Tick)):
             other = Timedelta(other)
@@ -553,13 +542,9 @@ def __truediv__(self, other):
             result = self._data / other
             return type(self)(result)
 
+    @unpack_zerodim_and_defer("__rtruediv__")
     def __rtruediv__(self, other):
         # X / timedelta is defined only for timedelta-like X
-        other = lib.item_from_zerodim(other)
-
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
-
         if isinstance(other, (timedelta, np.timedelta64, Tick)):
             other = Timedelta(other)
             if other is NaT:
@@ -599,11 +584,9 @@ def __rtruediv__(self, other):
                 f"Cannot divide {other.dtype} data by {type(self).__name__}"
             )
 
+    @unpack_zerodim_and_defer("__floordiv__")
     def __floordiv__(self, other):
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
 
-        other = lib.item_from_zerodim(other)
         if is_scalar(other):
             if isinstance(other, (timedelta, np.timedelta64, Tick)):
                 other = Timedelta(other)
@@ -665,11 +648,9 @@ def __floordiv__(self, other):
             dtype = getattr(other, "dtype", type(other).__name__)
             raise TypeError(f"Cannot divide {dtype} by {type(self).__name__}")
 
+    @unpack_zerodim_and_defer("__rfloordiv__")
     def __rfloordiv__(self, other):
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
 
-        other = lib.item_from_zerodim(other)
         if is_scalar(other):
             if isinstance(other, (timedelta, np.timedelta64, Tick)):
                 other = Timedelta(other)
@@ -714,45 +695,33 @@ def __rfloordiv__(self, other):
             dtype = getattr(other, "dtype", type(other).__name__)
             raise TypeError(f"Cannot divide {dtype} by {type(self).__name__}")
 
+    @unpack_zerodim_and_defer("__mod__")
     def __mod__(self, other):
         # Note: This is a naive implementation, can likely be optimized
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
-
-        other = lib.item_from_zerodim(other)
         if isinstance(other, (timedelta, np.timedelta64, Tick)):
             other = Timedelta(other)
         return self - (self // other) * other
 
+    @unpack_zerodim_and_defer("__rmod__")
     def __rmod__(self, other):
         # Note: This is a naive implementation, can likely be optimized
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
-
-        other = lib.item_from_zerodim(other)
         if isinstance(other, (timedelta, np.timedelta64, Tick)):
             other = Timedelta(other)
         return other - (other // self) * self
 
+    @unpack_zerodim_and_defer("__divmod__")
     def __divmod__(self, other):
         # Note: This is a naive implementation, can likely be optimized
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
-
-        other = lib.item_from_zerodim(other)
         if isinstance(other, (timedelta, np.timedelta64, Tick)):
             other = Timedelta(other)
 
         res1 = self // other
         res2 = self - res1 * other
         return res1, res2
 
+    @unpack_zerodim_and_defer("__rdivmod__")
     def __rdivmod__(self, other):
         # Note: This is a naive implementation, can likely be optimized
-        if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)):
-            return NotImplemented
-
-        other = lib.item_from_zerodim(other)
         if isinstance(other, (timedelta, np.timedelta64, Tick)):
             other = Timedelta(other)
 

pandas/core/construction.py

@@ -13,7 +13,7 @@
 
 from pandas._libs import lib
 from pandas._libs.tslibs import IncompatibleFrequency, OutOfBoundsDatetime
-from pandas._typing import ArrayLike, Dtype, DtypeObj
+from pandas._typing import AnyArrayLike, ArrayLike, Dtype, DtypeObj
 
 from pandas.core.dtypes.cast import (
     construct_1d_arraylike_from_scalar,
@@ -54,7 +54,9 @@
 
 
 def array(
-    data: Sequence[object], dtype: Optional[Dtype] = None, copy: bool = True,
+    data: Union[Sequence[object], AnyArrayLike],
+    dtype: Optional[Dtype] = None,
+    copy: bool = True,
 ) -> "ExtensionArray":
     """
     Create an array.

pandas/core/generic.py

@@ -9196,6 +9196,9 @@ def truncate(
             if before > after:
                 raise ValueError(f"Truncate: {after} must be after {before}")
 
+        if ax.is_monotonic_decreasing:
+            before, after = after, before
+
         slicer = [slice(None, None)] * self._AXIS_LEN
         slicer[axis] = slice(before, after)
         result = self.loc[tuple(slicer)]