Merge branch 'main' into dep_excel

Author: phofl

doc/source/reference/groupby.rst

@@ -65,7 +65,6 @@ Function application
 
    DataFrameGroupBy.all
    DataFrameGroupBy.any
-   DataFrameGroupBy.backfill
    DataFrameGroupBy.bfill
    DataFrameGroupBy.corr
    DataFrameGroupBy.corrwith
@@ -94,7 +93,6 @@ Function application
    DataFrameGroupBy.nth
    DataFrameGroupBy.nunique
    DataFrameGroupBy.ohlc
-   DataFrameGroupBy.pad
    DataFrameGroupBy.pct_change
    DataFrameGroupBy.prod
    DataFrameGroupBy.quantile
@@ -120,7 +118,6 @@ Function application
 
    SeriesGroupBy.all
    SeriesGroupBy.any
-   SeriesGroupBy.backfill
    SeriesGroupBy.bfill
    SeriesGroupBy.corr
    SeriesGroupBy.count
@@ -153,7 +150,6 @@ Function application
    SeriesGroupBy.nunique
    SeriesGroupBy.unique
    SeriesGroupBy.ohlc
-   SeriesGroupBy.pad
    SeriesGroupBy.pct_change
    SeriesGroupBy.prod
    SeriesGroupBy.quantile

doc/source/reference/resampling.rst

@@ -35,9 +35,7 @@ Upsampling
    :toctree: api/
 
    Resampler.ffill
-   Resampler.backfill
    Resampler.bfill
-   Resampler.pad
    Resampler.nearest
    Resampler.fillna
    Resampler.asfreq

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.5
 .. toctree::
    :maxdepth: 2
 
+   v1.5.2
    v1.5.1
    v1.5.0
 

doc/source/whatsnew/v1.5.0.rst

@@ -1246,4 +1246,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.4.4..v1.5.0|HEAD
+.. contributors:: v1.4.4..v1.5.0

doc/source/whatsnew/v1.5.1.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_151:
 
-What's new in 1.5.1 (October ??, 2022)
+What's new in 1.5.1 (October 19, 2022)
 --------------------------------------
 
 These are the changes in pandas 1.5.1. See :ref:`release` for a full changelog
@@ -111,12 +111,12 @@ Bug fixes
 Other
 ~~~~~
 - Avoid showing deprecated signatures when introspecting functions with warnings about arguments becoming keyword-only (:issue:`48692`)
--
--
 
 .. ---------------------------------------------------------------------------
 
 .. _whatsnew_151.contributors:
 
 Contributors
 ~~~~~~~~~~~~
+
+.. contributors:: v1.5.0..v1.5.1

doc/source/whatsnew/v1.5.2.rst

@@ -0,0 +1,41 @@
+.. _whatsnew_152:
+
+What's new in 1.5.2 (November ??, 2022)
+---------------------------------------
+
+These are the changes in pandas 1.5.2. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_152.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_152.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_152.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_152.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.5.1..v1.5.2|HEAD

doc/source/whatsnew/v2.0.0.rst

@@ -147,6 +147,7 @@ Removal of prior version deprecations/changes
 - Disallow passing non-round floats to :class:`Timestamp` with ``unit="M"`` or ``unit="Y"`` (:issue:`47266`)
 - Remove keywords ``convert_float`` and ``mangle_dupe_cols`` from :func:`read_excel` (:issue:`41176`)
 - Disallow passing non-keyword arguments to :func:`read_excel` except ``io`` and ``sheet_name`` (:issue:`34418`)
+- Remove :meth:`DataFrameGroupBy.pad` and :meth:`DataFrameGroupBy.backfill` (:issue:`45076`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.performance:
@@ -220,7 +221,7 @@ Conversion
 - Bug in :meth:`DataFrame.eval` incorrectly raising an ``AttributeError`` when there are negative values in function call (:issue:`46471`)
 - Bug in :meth:`Series.convert_dtypes` not converting dtype to nullable dtype when :class:`Series` contains ``NA`` and has dtype ``object`` (:issue:`48791`)
 - Bug where any :class:`ExtensionDtype` subclass with ``kind="M"`` would be interpreted as a timezone type (:issue:`34986`)
--
+- Bug in :class:`.arrays.ArrowExtensionArray` that would raise ``NotImplementedError`` when passed a sequence of strings or binary (:issue:`49172`)
 
 Strings
 ^^^^^^^
@@ -235,6 +236,7 @@ Interval
 Indexing
 ^^^^^^^^
 - Bug in :meth:`DataFrame.reindex` filling with wrong values when indexing columns and index for ``uint`` dtypes (:issue:`48184`)
+- Bug in :meth:`DataFrame.loc` coercing dtypes when setting values with a list indexer (:issue:`49159`)
 - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` when right hand side is :class:`DataFrame` with :class:`MultiIndex` columns (:issue:`49121`)
 - Bug in :meth:`DataFrame.reindex` casting dtype to ``object`` when :class:`DataFrame` has single extension array column when re-indexing ``columns`` and ``index`` (:issue:`48190`)
 - Bug in :func:`~DataFrame.describe` when formatting percentiles in the resulting index showed more decimals than needed (:issue:`46362`)
@@ -263,6 +265,7 @@ MultiIndex
 I/O
 ^^^
 - Bug in :func:`read_sas` caused fragmentation of :class:`DataFrame` and raised :class:`.errors.PerformanceWarning` (:issue:`48595`)
+- Bug in :func:`read_csv` for a single-line csv with fewer columns than ``names`` raised :class:`.errors.ParserError` with ``engine="c"`` (:issue:`47566`)
 -
 
 Period

pandas/_libs/parsers.pyx

@@ -744,6 +744,8 @@ cdef class TextReader:
         elif self.names is not None:
             # Names passed
             if self.parser.lines < 1:
+                if not self.has_usecols:
+                    self.parser.expected_fields = len(self.names)
                 self._tokenize_rows(1)
 
             header = [self.names]
@@ -756,6 +758,7 @@ cdef class TextReader:
             # Enforce this unless usecols
             if not self.has_usecols:
                 self.parser.expected_fields = max(field_count, len(self.names))
+
         else:
             # No header passed nor to be found in the file
             if self.parser.lines < 1:

pandas/_testing/__init__.py

@@ -201,8 +201,11 @@
     SIGNED_INT_PYARROW_DTYPES = [pa.int8(), pa.int16(), pa.int32(), pa.int64()]
     ALL_INT_PYARROW_DTYPES = UNSIGNED_INT_PYARROW_DTYPES + SIGNED_INT_PYARROW_DTYPES
 
+    # pa.float16 doesn't seem supported
+    # https://github.com/apache/arrow/blob/master/python/pyarrow/src/arrow/python/helpers.cc#L86
     FLOAT_PYARROW_DTYPES = [pa.float32(), pa.float64()]
-    STRING_PYARROW_DTYPES = [pa.string(), pa.utf8()]
+    STRING_PYARROW_DTYPES = [pa.string()]
+    BINARY_PYARROW_DTYPES = [pa.binary()]
 
     TIME_PYARROW_DTYPES = [
         pa.time32("s"),
@@ -225,6 +228,8 @@
     ALL_PYARROW_DTYPES = (
         ALL_INT_PYARROW_DTYPES
         + FLOAT_PYARROW_DTYPES
+        + STRING_PYARROW_DTYPES
+        + BINARY_PYARROW_DTYPES
         + TIME_PYARROW_DTYPES
         + DATE_PYARROW_DTYPES
         + DATETIME_PYARROW_DTYPES

pandas/core/arrays/arrow/array.py

@@ -220,8 +220,13 @@ def _from_sequence_of_strings(
         Construct a new ExtensionArray from a sequence of strings.
         """
         pa_type = to_pyarrow_type(dtype)
-        if pa_type is None:
-            # Let pyarrow try to infer or raise
+        if (
+            pa_type is None
+            or pa.types.is_binary(pa_type)
+            or pa.types.is_string(pa_type)
+        ):
+            # pa_type is None: Let pa.array infer
+            # pa_type is string/binary: scalars already correct type
             scalars = strings
         elif pa.types.is_timestamp(pa_type):
             from pandas.core.tools.datetimes import to_datetime

pandas/core/groupby/base.py

@@ -73,7 +73,6 @@ def maybe_normalize_deprecated_kernels(kernel) -> Literal["bfill", "ffill"]:
 
 transformation_kernels = frozenset(
     [
-        "backfill",
         "bfill",
         "cumcount",
         "cummax",
@@ -84,7 +83,6 @@ def maybe_normalize_deprecated_kernels(kernel) -> Literal["bfill", "ffill"]:
         "ffill",
         "fillna",
         "ngroup",
-        "pad",
         "pct_change",
         "rank",
         "shift",

pandas/core/groupby/groupby.py

@@ -2936,31 +2936,6 @@ def ffill(self, limit=None):
         """
         return self._fill("ffill", limit=limit)
 
-    def pad(self, limit=None):
-        """
-        Forward fill the values.
-
-        .. deprecated:: 1.4
-            Use ffill instead.
-
-        Parameters
-        ----------
-        limit : int, optional
-            Limit of how many values to fill.
-
-        Returns
-        -------
-        Series or DataFrame
-            Object with missing values filled.
-        """
-        warnings.warn(
-            "pad is deprecated and will be removed in a future version. "
-            "Use ffill instead.",
-            FutureWarning,
-            stacklevel=find_stack_level(),
-        )
-        return self.ffill(limit=limit)
-
     @final
     @Substitution(name="groupby")
     def bfill(self, limit=None):
@@ -2986,31 +2961,6 @@ def bfill(self, limit=None):
         """
         return self._fill("bfill", limit=limit)
 
-    def backfill(self, limit=None):
-        """
-        Backward fill the values.
-
-        .. deprecated:: 1.4
-            Use bfill instead.
-
-        Parameters
-        ----------
-        limit : int, optional
-            Limit of how many values to fill.
-
-        Returns
-        -------
-        Series or DataFrame
-            Object with missing values filled.
-        """
-        warnings.warn(
-            "backfill is deprecated and will be removed in a future version. "
-            "Use bfill instead.",
-            FutureWarning,
-            stacklevel=find_stack_level(),
-        )
-        return self.bfill(limit=limit)
-
     @final
     @Substitution(name="groupby")
     @Substitution(see_also=_common_see_also)

pandas/core/indexing.py

@@ -1898,16 +1898,20 @@ def _setitem_with_indexer_2d_value(self, indexer, value):
 
         ilocs = self._ensure_iterable_column_indexer(indexer[1])
 
-        # GH#7551 Note that this coerces the dtype if we are mixed
-        value = np.array(value, dtype=object)
+        if not is_array_like(value):
+            # cast lists to array
+            value = np.array(value, dtype=object)
         if len(ilocs) != value.shape[1]:
             raise ValueError(
                 "Must have equal len keys and value when setting with an ndarray"
             )
 
         for i, loc in enumerate(ilocs):
-            # setting with a list, re-coerces
-            self._setitem_single_column(loc, value[:, i].tolist(), pi)
+            value_col = value[:, i]
+            if is_object_dtype(value_col.dtype):
+                # casting to list so that we do type inference in setitem_single_column
+                value_col = value_col.tolist()
+            self._setitem_single_column(loc, value_col, pi)
 
     def _setitem_with_indexer_frame_value(self, indexer, value: DataFrame, name: str):
         ilocs = self._ensure_iterable_column_indexer(indexer[1])

pandas/core/resample.py

@@ -11,7 +11,6 @@
     final,
     no_type_check,
 )
-import warnings
 
 import numpy as np
 
@@ -50,7 +49,6 @@
     deprecate_nonkeyword_arguments,
     doc,
 )
-from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.generic import (
     ABCDataFrame,
@@ -562,30 +560,6 @@ def ffill(self, limit=None):
         """
         return self._upsample("ffill", limit=limit)
 
-    def pad(self, limit=None):
-        """
-        Forward fill the values.
-
-        .. deprecated:: 1.4
-            Use ffill instead.
-
-        Parameters
-        ----------
-        limit : int, optional
-            Limit of how many values to fill.
-
-        Returns
-        -------
-        An upsampled Series.
-        """
-        warnings.warn(
-            "pad is deprecated and will be removed in a future version. "
-            "Use ffill instead.",
-            FutureWarning,
-            stacklevel=find_stack_level(),
-        )
-        return self.ffill(limit=limit)
-
     def nearest(self, limit=None):
         """
         Resample by using the nearest value.
@@ -748,31 +722,6 @@ def bfill(self, limit=None):
         """
         return self._upsample("bfill", limit=limit)
 
-    def backfill(self, limit=None):
-        """
-        Backward fill the values.
-
-        .. deprecated:: 1.4
-            Use bfill instead.
-
-        Parameters
-        ----------
-        limit : int, optional
-            Limit of how many values to fill.
-
-        Returns
-        -------
-        Series, DataFrame
-            An upsampled Series or DataFrame with backward filled NaN values.
-        """
-        warnings.warn(
-            "backfill is deprecated and will be removed in a future version. "
-            "Use bfill instead.",
-            FutureWarning,
-            stacklevel=find_stack_level(),
-        )
-        return self.bfill(limit=limit)
-
     def fillna(self, method, limit=None):
         """
         Fill missing values introduced by upsampling.