Merge remote-tracking branch 'upstream/main' into enh/parquet/arrow_fs

Author: mroeschke

asv_bench/benchmarks/frame_methods.py

@@ -444,6 +444,22 @@ def time_dropna_axis_mixed_dtypes(self, how, axis):
         self.df_mixed.dropna(how=how, axis=axis)
 
 
+class Isna:
+    params = ["float64", "Float64", "float64[pyarrow]"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        data = np.random.randn(10000, 1000)
+        # all-na columns
+        data[:, 600:800] = np.nan
+        # partial-na columns
+        data[800:1000, 4000:5000] = np.nan
+        self.df = DataFrame(data, dtype=dtype)
+
+    def time_isna(self, dtype):
+        self.df.isna()
+
+
 class Count:
     params = [0, 1]
     param_names = ["axis"]

doc/source/whatsnew/v2.1.0.rst

@@ -103,6 +103,7 @@ Deprecations
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
+- Performance improvement in :meth:`~arrays.ArrowExtensionArray.isna` when array has zero nulls or is all nulls (:issue:`51630`)
 - Performance improvement in :meth:`DataFrame.first_valid_index` and :meth:`DataFrame.last_valid_index` for extension array dtypes (:issue:`51549`)
 - Performance improvement in :meth:`DataFrame.clip` and :meth:`Series.clip` (:issue:`51472`)
 - Performance improvement in :func:`read_parquet` on string columns when using ``use_nullable_dtypes=True`` (:issue:`47345`)
@@ -121,12 +122,13 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
--
+- Bug in :meth:`Timestamp.round` with values close to the implementation bounds returning incorrect results instead of raising ``OutOfBoundsDatetime`` (:issue:`51494`)
 -
 
 Timedelta
 ^^^^^^^^^
--
+- Bug in :meth:`Timedelta.round` with values close to the implementation bounds returning incorrect results instead of raising ``OutOfBoundsTimedelta`` (:issue:`51494`)
+- Bug in :class:`TimedeltaIndex` division or multiplication leading to ``.freq`` of "0 Days" instead of ``None`` (:issue:`51575`)
 -
 
 Timezones
@@ -186,7 +188,7 @@ Plotting
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
--
+- Bug in :meth:`DataFrameGroupBy.idxmin`, :meth:`SeriesGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmax` return wrong dtype when used on empty DataFrameGroupBy or SeriesGroupBy (:issue:`51423`)
 -
 
 Reshaping

pandas/_libs/lib.pyi

@@ -71,6 +71,7 @@ def maybe_convert_objects(
     *,
     try_float: bool = ...,
     safe: bool = ...,
+    convert_numeric: bool = ...,
     convert_datetime: Literal[False] = ...,
     convert_timedelta: Literal[False] = ...,
     convert_period: Literal[False] = ...,
@@ -84,6 +85,7 @@ def maybe_convert_objects(
     *,
     try_float: bool = ...,
     safe: bool = ...,
+    convert_numeric: bool = ...,
     convert_datetime: Literal[False] = ...,
     convert_timedelta: bool = ...,
     convert_period: Literal[False] = ...,
@@ -97,6 +99,7 @@ def maybe_convert_objects(
     *,
     try_float: bool = ...,
     safe: bool = ...,
+    convert_numeric: bool = ...,
     convert_datetime: bool = ...,
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
@@ -110,6 +113,7 @@ def maybe_convert_objects(
     *,
     try_float: bool = ...,
     safe: bool = ...,
+    convert_numeric: bool = ...,
     convert_datetime: Literal[True] = ...,
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
@@ -123,6 +127,7 @@ def maybe_convert_objects(
     *,
     try_float: bool = ...,
     safe: bool = ...,
+    convert_numeric: bool = ...,
     convert_datetime: bool = ...,
     convert_timedelta: bool = ...,
     convert_period: Literal[True] = ...,
@@ -136,6 +141,7 @@ def maybe_convert_objects(
     *,
     try_float: bool = ...,
     safe: bool = ...,
+    convert_numeric: bool = ...,
     convert_datetime: bool = ...,
     convert_timedelta: bool = ...,
     convert_period: bool = ...,

pandas/_libs/lib.pyx

@@ -2367,6 +2367,7 @@ def maybe_convert_objects(ndarray[object] objects,
                           *,
                           bint try_float=False,
                           bint safe=False,
+                          bint convert_numeric=True,  # NB: different default!
                           bint convert_datetime=False,
                           bint convert_timedelta=False,
                           bint convert_period=False,
@@ -2386,6 +2387,8 @@ def maybe_convert_objects(ndarray[object] objects,
     safe : bool, default False
         Whether to upcast numeric type (e.g. int cast to float). If set to
         True, no upcasting will be performed.
+    convert_numeric : bool, default True
+        Whether to convert numeric entries.
     convert_datetime : bool, default False
         If an array-like object contains only datetime values or NaT is
         encountered, whether to convert and return an array of M8[ns] dtype.
@@ -2463,9 +2466,13 @@ def maybe_convert_objects(ndarray[object] objects,
         elif util.is_bool_object(val):
             seen.bool_ = True
             bools[i] = val
+            if not convert_numeric:
+                break
         elif util.is_float_object(val):
             floats[i] = complexes[i] = val
             seen.float_ = True
+            if not convert_numeric:
+                break
         elif is_timedelta(val):
             if convert_timedelta:
                 seen.timedelta_ = True
@@ -2497,10 +2504,14 @@ def maybe_convert_objects(ndarray[object] objects,
                 else:
                     uints[i] = val
                     ints[i] = val
+            if not convert_numeric:
+                break
 
         elif util.is_complex_object(val):
             complexes[i] = val
             seen.complex_ = True
+            if not convert_numeric:
+                break
         elif PyDateTime_Check(val) or util.is_datetime64_object(val):
 
             # if we have an tz's attached then return the objects
@@ -2638,6 +2649,12 @@ def maybe_convert_objects(ndarray[object] objects,
         else:
             seen.object_ = True
 
+    if not convert_numeric:
+        # Note: we count "bool" as numeric here. This is becase
+        #  np.array(list_of_items) will convert bools just like it will numeric
+        #  entries.
+        return objects
+
     if seen.bool_:
         if seen.is_bool:
             # is_bool property rules out everything else

pandas/_libs/tslibs/fields.pyx

@@ -704,7 +704,7 @@ cdef ndarray[int64_t] _ceil_int64(const int64_t[:] values, int64_t unit):
     cdef:
         Py_ssize_t i, n = len(values)
         ndarray[int64_t] result = np.empty(n, dtype="i8")
-        int64_t res, value
+        int64_t res, value, remainder
 
     with cython.overflowcheck(True):
         for i in range(n):
@@ -732,6 +732,34 @@ cdef ndarray[int64_t] _roundup_int64(values, int64_t unit):
     return _floor_int64(values + unit // 2, unit)
 
 
+cdef ndarray[int64_t] _round_nearest_int64(const int64_t[:] values, int64_t unit):
+    cdef:
+        Py_ssize_t i, n = len(values)
+        ndarray[int64_t] result = np.empty(n, dtype="i8")
+        int64_t res, value, half, remainder, quotient
+
+    half = unit // 2
+
+    with cython.overflowcheck(True):
+        for i in range(n):
+            value = values[i]
+
+            if value == NPY_NAT:
+                res = NPY_NAT
+            else:
+                quotient, remainder = divmod(value, unit)
+                if remainder > half:
+                    res = value + (unit - remainder)
+                elif remainder == half and quotient % 2:
+                    res = value + (unit - remainder)
+                else:
+                    res = value - remainder
+
+            result[i] = res
+
+    return result
+
+
 def round_nsint64(values: np.ndarray, mode: RoundTo, nanos: int) -> np.ndarray:
     """
     Applies rounding mode at given frequency
@@ -762,13 +790,7 @@ def round_nsint64(values: np.ndarray, mode: RoundTo, nanos: int) -> np.ndarray:
         # for odd unit there is no need of a tie break
         if unit % 2:
             return _rounddown_int64(values, unit)
-        quotient, remainder = np.divmod(values, unit)
-        mask = np.logical_or(
-            remainder > (unit // 2),
-            np.logical_and(remainder == (unit // 2), quotient % 2)
-        )
-        quotient[mask] += 1
-        return quotient * unit
+        return _round_nearest_int64(values, unit)
 
     # if/elif above should catch all rounding modes defined in enum 'RoundTo':
     # if flow of control arrives here, it is a bug

pandas/_libs/tslibs/timedeltas.pyx

@@ -1824,7 +1824,12 @@ class Timedelta(_Timedelta):
         unit = delta_to_nanoseconds(to_offset(freq), self._creso)
 
         arr = np.array([self._value], dtype="i8")
-        result = round_nsint64(arr, mode, unit)[0]
+        try:
+            result = round_nsint64(arr, mode, unit)[0]
+        except OverflowError as err:
+            raise OutOfBoundsTimedelta(
+                f"Cannot round {self} to freq={freq} without overflow"
+            ) from err
         return Timedelta._from_value_and_reso(result, self._creso)
 
     def round(self, freq):

pandas/_libs/tslibs/timestamps.pyx

@@ -1692,7 +1692,13 @@ class Timestamp(_Timestamp):
         value = np.array([value], dtype=np.int64)
 
         # Will only ever contain 1 element for timestamp
-        r = round_nsint64(value, mode, nanos)[0]
+        try:
+            r = round_nsint64(value, mode, nanos)[0]
+        except OverflowError as err:
+            raise OutOfBoundsDatetime(
+                f"Cannot round {self} to freq={freq} without overflow"
+            ) from err
+
         result = Timestamp._from_value_and_reso(r, self._creso, None)
         if self.tz is not None:
             result = result.tz_localize(

pandas/core/arrays/arrow/array.py

@@ -557,6 +557,13 @@ def isna(self) -> npt.NDArray[np.bool_]:
 
         This should return a 1-D array the same length as 'self'.
         """
+        # GH51630: fast paths
+        null_count = self._data.null_count
+        if null_count == 0:
+            return np.zeros(len(self), dtype=np.bool_)
+        elif null_count == len(self):
+            return np.ones(len(self), dtype=np.bool_)
+
         return self._data.is_null().to_numpy()
 
     def argsort(

pandas/core/arrays/timedeltas.py

@@ -467,6 +467,9 @@ def __mul__(self, other) -> TimedeltaArray:
             freq = None
             if self.freq is not None and not isna(other):
                 freq = self.freq * other
+                if freq.n == 0:
+                    # GH#51575 Better to have no freq than an incorrect one
+                    freq = None
             return type(self)._simple_new(result, dtype=result.dtype, freq=freq)
 
         if not hasattr(other, "dtype"):
@@ -526,17 +529,10 @@ def _scalar_divlike_op(self, other, op):
                 # Note: freq gets division, not floor-division, even if op
                 #  is floordiv.
                 freq = self.freq / other
-
-                # TODO: 2022-12-24 test_ufunc_coercions, test_tdi_ops_attributes
-                #  get here for truediv, no tests for floordiv
-
-                if op is operator.floordiv:
-                    if freq.nanos == 0 and self.freq.nanos != 0:
-                        # e.g. if self.freq is Nano(1) then dividing by 2
-                        #  rounds down to zero
-                        # TODO: 2022-12-24 should implement the same check
-                        #  for truediv case
-                        freq = None
+                if freq.nanos == 0 and self.freq.nanos != 0:
+                    # e.g. if self.freq is Nano(1) then dividing by 2
+                    #  rounds down to zero
+                    freq = None
 
             return type(self)._simple_new(result, dtype=result.dtype, freq=freq)
 

pandas/core/dtypes/cast.py

@@ -1156,23 +1156,20 @@ def maybe_infer_to_datetimelike(
     if not len(value):
         return value
 
-    out = lib.maybe_convert_objects(
+    # error: Incompatible return value type (got "Union[ExtensionArray,
+    # ndarray[Any, Any]]", expected "Union[ndarray[Any, Any], DatetimeArray,
+    # TimedeltaArray, PeriodArray, IntervalArray]")
+    return lib.maybe_convert_objects(  # type: ignore[return-value]
         value,
+        # Here we do not convert numeric dtypes, as if we wanted that,
+        #  numpy would have done it for us.
+        convert_numeric=False,
         convert_period=True,
         convert_interval=True,
         convert_timedelta=True,
         convert_datetime=True,
         dtype_if_all_nat=np.dtype("M8[ns]"),
     )
-    if out.dtype.kind in ["i", "u", "f", "b", "c"]:
-        # Here we do not convert numeric dtypes, as if we wanted that,
-        #  numpy would have done it for us.
-        #  See also _maybe_cast_data_without_dtype
-        return value
-    # Incompatible return value type (got "Union[ExtensionArray, ndarray[Any, Any]]",
-    # expected "Union[ndarray[Any, Any], DatetimeArray, TimedeltaArray, PeriodArray,
-    # IntervalArray]")
-    return out  # type: ignore[return-value]
 
 
 def maybe_cast_to_datetime(

pandas/core/frame.py

@@ -2801,8 +2801,12 @@ def to_parquet(
             ``io.parquet.engine`` is used. The default ``io.parquet.engine``
             behavior is to try 'pyarrow', falling back to 'fastparquet' if
             'pyarrow' is unavailable.
-        compression : {{'snappy', 'gzip', 'brotli', None}}, default 'snappy'
+        compression : str or None, default 'snappy'
             Name of the compression to use. Use ``None`` for no compression.
+            The supported compression methods actually depend on which engine
+            is used. For 'pyarrow', 'snappy', 'gzip', 'brotli', 'lz4', 'zstd'
+            are all supported. For 'fastparquet', only 'gzip' and 'snappy' are
+            supported.
         index : bool, default None
             If ``True``, include the dataframe's index(es) in the file output.
             If ``False``, they will not be written to the file.
@@ -8211,7 +8215,7 @@ def groupby(
         level: IndexLabel | None = None,
         as_index: bool = True,
         sort: bool = True,
-        group_keys: bool | lib.NoDefault = no_default,
+        group_keys: bool = True,
         observed: bool = False,
         dropna: bool = True,
     ) -> DataFrameGroupBy:

pandas/core/generic.py

@@ -6885,8 +6885,10 @@ def fillna(
                         "with dict/Series column "
                         "by column"
                     )
-
-                result = self if inplace else self.copy()
+                if using_copy_on_write():
+                    result = self.copy(deep=None)
+                else:
+                    result = self if inplace else self.copy()
                 is_dict = isinstance(downcast, dict)
                 for k, v in value.items():
                     if k not in result:
@@ -6940,8 +6942,10 @@ def fillna(
                                     result.iloc[:, loc] = res_loc
                                 else:
                                     result.isetitem(loc, res_loc)
-
-                return result if not inplace else None
+                if inplace:
+                    return self._update_inplace(result)
+                else:
+                    return result
 
             elif not is_list_like(value):
                 if axis == 1: