Merge remote-tracking branch 'upstream/master' into ci/simplify_deps

Author: mroeschke

doc/source/_static/style/latex_stocks.png

@@ -27,6 +27,7 @@ Styler properties
    Styler.template_html_style
    Styler.template_html_table
    Styler.template_latex
+   Styler.template_string
    Styler.loader
 
 Style application
@@ -74,5 +75,6 @@ Style export and import
    Styler.to_html
    Styler.to_latex
    Styler.to_excel
+   Styler.to_string
    Styler.export
    Styler.use

doc/source/_static/style/latex_stocks_html.png

@@ -1305,14 +1305,38 @@ You can elect to skip bad lines:
     0  1  2   3
     1  8  9  10
 
+Or pass a callable function to handle the bad line if ``engine="python"``.
+The bad line will be a list of strings that was split by the ``sep``:
+
+.. code-block:: ipython
+
+    In [29]: external_list = []
+
+    In [30]: def bad_lines_func(line):
+        ...:     external_list.append(line)
+        ...:     return line[-3:]
+
+    In [31]: pd.read_csv(StringIO(data), on_bad_lines=bad_lines_func, engine="python")
+    Out[31]:
+       a  b   c
+    0  1  2   3
+    1  5  6   7
+    2  8  9  10
+
+    In [32]: external_list
+    Out[32]: [4, 5, 6, 7]
+
+    .. versionadded:: 1.4.0
+
+
 You can also use the ``usecols`` parameter to eliminate extraneous column
 data that appear in some lines but not others:
 
 .. code-block:: ipython
 
-   In [30]: pd.read_csv(StringIO(data), usecols=[0, 1, 2])
+   In [33]: pd.read_csv(StringIO(data), usecols=[0, 1, 2])
 
-    Out[30]:
+    Out[33]:
        a  b   c
     0  1  2   3
     1  4  5   6
@@ -1324,9 +1348,9 @@ fields are filled with ``NaN``.
 
 .. code-block:: ipython
 
-   In [31]: pd.read_csv(StringIO(data), names=['a', 'b', 'c', 'd'])
+   In [34]: pd.read_csv(StringIO(data), names=['a', 'b', 'c', 'd'])
 
-   Out[31]:
+   Out[34]:
        a  b   c  d
     0  1  2   3  NaN
     1  4  5   6  7

doc/source/reference/style.rst

@@ -208,6 +208,7 @@ Other enhancements
 - :meth:`Series.str.split` now supports a ``regex`` argument that explicitly specifies whether the pattern is a regular expression. Default is ``None`` (:issue:`43563`, :issue:`32835`, :issue:`25549`)
 - :meth:`DataFrame.dropna` now accepts a single label as ``subset`` along with array-like (:issue:`41021`)
 - Added :meth:`DataFrameGroupBy.value_counts` (:issue:`43564`)
+- :func:`read_csv` now accepts a ``callable`` function in ``on_bad_lines`` when ``engine="python"`` for custom handling of bad lines (:issue:`5686`)
 - :class:`ExcelWriter` argument ``if_sheet_exists="overlay"`` option added (:issue:`40231`)
 - :meth:`read_excel` now accepts a ``decimal`` argument that allow the user to specify the decimal point when parsing string columns to numeric (:issue:`14403`)
 - :meth:`.GroupBy.mean`, :meth:`.GroupBy.std`, :meth:`.GroupBy.var`, :meth:`.GroupBy.sum` now supports `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`43731`, :issue:`44862`, :issue:`44939`)
@@ -815,7 +816,7 @@ Indexing
 
 Missing
 ^^^^^^^
-- Bug in :meth:`DataFrame.fillna` with limit and no method ignores axis='columns' or ``axis = 1`` (:issue:`40989`)
+- Bug in :meth:`DataFrame.fillna` with limit and no method ignores axis='columns' or ``axis = 1`` (:issue:`40989`, :issue:`17399`)
 - Bug in :meth:`DataFrame.fillna` not replacing missing values when using a dict-like ``value`` and duplicate column names (:issue:`43476`)
 - Bug in constructing a :class:`DataFrame` with a dictionary ``np.datetime64`` as a value and ``dtype='timedelta64[ns]'``, or vice-versa, incorrectly casting instead of raising (:issue:`44428`)
 - Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` with ``inplace=True`` not writing to the underlying array(s) in-place (:issue:`44749`)

doc/source/user_guide/io.rst

@@ -14,10 +14,12 @@ including other versions of pandas.
 Enhancements
 ~~~~~~~~~~~~
 
-.. _whatsnew_150.enhancements.enhancement1:
+.. _whatsnew_150.enhancements.styler:
 
-enhancement1
-^^^^^^^^^^^^
+Styler
+^^^^^^
+
+  - New method :meth:`.Styler.to_string` for alternative customisable output methods (:issue:`44502`)
 
 .. _whatsnew_150.enhancements.enhancement2:
 
@@ -151,7 +153,7 @@ Interval
 
 Indexing
 ^^^^^^^^
--
+- Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
 -
 
 Missing

doc/source/whatsnew/v1.4.0.rst

@@ -714,7 +714,7 @@ cpdef ndarray[object] ensure_string_array(
             return out
 
         arr = arr.to_numpy()
-    elif not isinstance(arr, np.ndarray):
+    elif not util.is_array(arr):
         arr = np.array(arr, dtype="object")
 
     result = np.asarray(arr, dtype="object")
@@ -729,7 +729,7 @@ cpdef ndarray[object] ensure_string_array(
             continue
 
         if not checknull(val):
-            if not isinstance(val, np.floating):
+            if not util.is_float_object(val):
                 # f"{val}" is faster than str(val)
                 result[i] = f"{val}"
             else:

doc/source/whatsnew/v1.5.0.rst

@@ -1251,10 +1251,29 @@ class Timedelta(_Timedelta):
 
     Notes
     -----
+    The constructor may take in either both values of value and unit or
+    kwargs as above. Either one of them must be used during initialization
+
     The ``.value`` attribute is always in ns.
 
     If the precision is higher than nanoseconds, the precision of the duration is
     truncated to nanoseconds.
+
+    Examples
+    --------
+    Here we initialize Timedelta object with both value and unit
+
+    >>> td = pd.Timedelta(1, "d")
+    >>> td
+    Timedelta('1 days 00:00:00')
+
+    Here we initialize the Timedelta object with kwargs
+
+    >>> td2 = pd.Timedelta(days=1)
+    >>> td2
+    Timedelta('1 days 00:00:00')
+
+    We see that either way we get the same result
     """
 
     _req_any_kwargs_new = {"weeks", "days", "hours", "minutes", "seconds",

pandas/_libs/lib.pyx

@@ -1360,7 +1360,7 @@ def remove_unused_categories(self, inplace=no_default):
 
     def map(self, mapper):
         """
-        Map categories using input correspondence (dict, Series, or function).
+        Map categories using an input mapping or function.
 
         Maps the categories to new categories. If the mapping correspondence is
         one-to-one the result is a :class:`~pandas.Categorical` which has the
@@ -1994,7 +1994,7 @@ def _formatter(self, boxed: bool = False):
         # Defer to CategoricalFormatter's formatter.
         return None
 
-    def _tidy_repr(self, max_vals=10, footer=True) -> str:
+    def _tidy_repr(self, max_vals: int = 10, footer: bool = True) -> str:
         """
         a short repr displaying only max_vals and an optional (but default
         footer)
@@ -2009,7 +2009,7 @@ def _tidy_repr(self, max_vals=10, footer=True) -> str:
 
         return str(result)
 
-    def _repr_categories(self):
+    def _repr_categories(self) -> list[str]:
         """
         return the base repr for the categories
         """

pandas/_libs/tslibs/timedeltas.pyx

@@ -41,7 +41,7 @@
 from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_inclusive
 
-from pandas.core.dtypes.cast import astype_dt64_to_dt64tz
+from pandas.core.dtypes.astype import astype_dt64_to_dt64tz
 from pandas.core.dtypes.common import (
     DT64NS_DTYPE,
     INT64_DTYPE,
@@ -570,7 +570,7 @@ def dtype(self) -> np.dtype | DatetimeTZDtype:  # type: ignore[override]
     @property
     def tz(self) -> tzinfo | None:
         """
-        Return timezone, if any.
+        Return the timezone.
 
         Returns
         -------
@@ -1278,7 +1278,9 @@ def day_name(self, locale=None):
     @property
     def time(self) -> npt.NDArray[np.object_]:
         """
-        Returns numpy array of datetime.time. The time part of the Timestamps.
+        Returns numpy array of :class:`datetime.time` objects.
+
+        The time part of the Timestamps.
         """
         # If the Timestamps have a timezone that is not UTC,
         # convert them into their i8 representation while
@@ -1290,16 +1292,20 @@ def time(self) -> npt.NDArray[np.object_]:
     @property
     def timetz(self) -> npt.NDArray[np.object_]:
         """
-        Returns numpy array of datetime.time also containing timezone
-        information. The time part of the Timestamps.
+        Returns numpy array of :class:`datetime.time` objects with timezone
+        information.
+
+        The time part of the Timestamps.
         """
         return ints_to_pydatetime(self.asi8, self.tz, box="time")
 
     @property
     def date(self) -> npt.NDArray[np.object_]:
         """
-        Returns numpy array of python datetime.date objects (namely, the date
-        part of Timestamps without timezone information).
+        Returns numpy array of python :class:`datetime.date` objects.
+
+        Namely, the date part of Timestamps without time and
+        timezone information.
         """
         # If the Timestamps have a timezone that is not UTC,
         # convert them into their i8 representation while

pandas/core/arrays/categorical.py

@@ -16,7 +16,7 @@
 )
 from pandas.util._decorators import cache_readonly
 
-from pandas.core.dtypes.cast import astype_nansafe
+from pandas.core.dtypes.astype import astype_nansafe
 from pandas.core.dtypes.common import (
     is_bool_dtype,
     is_datetime64_dtype,

pandas/core/arrays/datetimes.py

@@ -767,7 +767,7 @@ def _quantile(
         We assume that all impacted cases are 1D-only.
         """
         mask = np.atleast_2d(np.asarray(self.isna()))
-        npvalues = np.atleast_2d(np.asarray(self))
+        npvalues: np.ndarray = np.atleast_2d(np.asarray(self))
 
         res = quantile_with_mask(
             npvalues,

pandas/core/arrays/floating.py

@@ -44,8 +44,8 @@
 from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_insert_loc
 
+from pandas.core.dtypes.astype import astype_nansafe
 from pandas.core.dtypes.cast import (
-    astype_nansafe,
     construct_1d_arraylike_from_scalar,
     find_common_type,
     maybe_box_datetimelike,
@@ -298,7 +298,7 @@ class SparseArray(OpsMixin, PandasObject, ExtensionArray):
             repeats of the scalar value instead.
 
     fill_value : scalar, optional
-        Elements in `data` that are `fill_value` are not stored in the
+        Elements in data that are ``fill_value`` are not stored in the
         SparseArray. For memory savings, this should be the most common value
         in `data`. By default, `fill_value` depends on the dtype of `data`:
 
@@ -1265,7 +1265,7 @@ def astype(self, dtype: AstypeArg | None = None, copy: bool = True):
 
     def map(self: SparseArrayT, mapper) -> SparseArrayT:
         """
-        Map categories using input correspondence (dict, Series, or function).
+        Map categories using an input mapping or function.
 
         Parameters
         ----------

pandas/core/arrays/masked.py

@@ -18,11 +18,11 @@
 from pandas.errors import PerformanceWarning
 from pandas.util._exceptions import find_stack_level
 
+from pandas.core.dtypes.astype import astype_nansafe
 from pandas.core.dtypes.base import (
     ExtensionDtype,
     register_extension_dtype,
 )
-from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.common import (
     is_bool_dtype,
     is_object_dtype,

pandas/core/arrays/sparse/array.py

@@ -38,7 +38,7 @@
 from pandas.compat.numpy import function as nv
 from pandas.util._validators import validate_endpoints
 
-from pandas.core.dtypes.cast import astype_td64_unit_conversion
+from pandas.core.dtypes.astype import astype_td64_unit_conversion
 from pandas.core.dtypes.common import (
     DT64NS_DTYPE,
     TD64NS_DTYPE,