Merge remote-tracking branch 'upstream/master' into deprecate-nonkeyword-args-reset_index

Author: MarcoGorelli

asv_bench/benchmarks/io/style.py

@@ -20,19 +20,19 @@ def setup(self, cols, rows):
 
     def time_apply_render(self, cols, rows):
         self._style_apply()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def peakmem_apply_render(self, cols, rows):
         self._style_apply()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def time_classes_render(self, cols, rows):
         self._style_classes()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def peakmem_classes_render(self, cols, rows):
         self._style_classes()
-        self.st._render_html()
+        self.st._render_html(True, True)
 
     def time_format_render(self, cols, rows):
         self._style_format()

ci/deps/actions-37-minimum_versions.yaml

@@ -17,7 +17,7 @@ dependencies:
   - bottleneck=1.2.1
   - jinja2=2.10
   - numba=0.46.0
-  - numexpr=2.6.8
+  - numexpr=2.7.0
   - numpy=1.17.3
   - openpyxl=3.0.0
   - pytables=3.5.1

doc/source/getting_started/install.rst

@@ -234,7 +234,7 @@ Recommended dependencies
 
 * `numexpr <https://github.com/pydata/numexpr>`__: for accelerating certain numerical operations.
   ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups.
-  If installed, must be Version 2.6.8 or higher.
+  If installed, must be Version 2.7.0 or higher.
 
 * `bottleneck <https://github.com/pydata/bottleneck>`__: for accelerating certain types of ``nan``
   evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups. If installed,

doc/source/user_guide/options.rst

@@ -482,6 +482,11 @@ plotting.backend                        matplotlib   Change the plotting backend
                                                      like Bokeh, Altair, etc.
 plotting.matplotlib.register_converters True         Register custom converters with
                                                      matplotlib. Set to False to de-register.
+styler.sparse.index                     True         "Sparsify" MultiIndex display for rows
+                                                     in Styler output (don't display repeated
+                                                     elements in outer levels within groups).
+styler.sparse.columns                   True         "Sparsify" MultiIndex display for columns
+                                                     in Styler output.
 ======================================= ============ ==================================
 
 

doc/source/whatsnew/v1.2.0.rst

@@ -381,6 +381,7 @@ this pathological behavior (:issue:`37827`):
 *New behavior*:
 
 .. ipython:: python
+   :okwarning:
 
     df.mean()
 
@@ -394,6 +395,7 @@ instead of casting to a NumPy array which may have different semantics (:issue:`
 :issue:`28949`, :issue:`21020`).
 
 .. ipython:: python
+   :okwarning:
 
     ser = pd.Series([0, 1], dtype="category", name="A")
     df = ser.to_frame()
@@ -411,6 +413,7 @@ instead of casting to a NumPy array which may have different semantics (:issue:`
 *New behavior*:
 
 .. ipython:: python
+   :okwarning:
 
     df.any()
 

doc/source/whatsnew/v1.3.0.rst

@@ -139,6 +139,7 @@ precision, and perform HTML escaping (:issue:`40437` :issue:`40134`). There have
 properly format HTML and eliminate some inconsistencies (:issue:`39942` :issue:`40356` :issue:`39807` :issue:`39889` :issue:`39627`)
 
 :class:`.Styler` has also been compatible with non-unique index or columns, at least for as many features as are fully compatible, others made only partially compatible (:issue:`41269`).
+One also has greater control of the display through separate sparsification of the index or columns, using the new 'styler' options context (:issue:`41142`).
 
 Documentation has also seen major revisions in light of new features (:issue:`39720` :issue:`39317` :issue:`40493`)
 
@@ -197,7 +198,7 @@ Other enhancements
 - Improved integer type mapping from pandas to SQLAlchemy when using :meth:`DataFrame.to_sql` (:issue:`35076`)
 - :func:`to_numeric` now supports downcasting of nullable ``ExtensionDtype`` objects (:issue:`33013`)
 - Add support for dict-like names in :class:`MultiIndex.set_names` and :class:`MultiIndex.rename` (:issue:`20421`)
-- :func:`pandas.read_excel` can now auto detect .xlsb files (:issue:`35416`)
+- :func:`pandas.read_excel` can now auto detect .xlsb files and older .xls files (:issue:`35416`, :issue:`41225`)
 - :class:`pandas.ExcelWriter` now accepts an ``if_sheet_exists`` parameter to control the behaviour of append mode when writing to existing sheets (:issue:`40230`)
 - :meth:`.Rolling.sum`, :meth:`.Expanding.sum`, :meth:`.Rolling.mean`, :meth:`.Expanding.mean`, :meth:`.ExponentialMovingWindow.mean`, :meth:`.Rolling.median`, :meth:`.Expanding.median`, :meth:`.Rolling.max`, :meth:`.Expanding.max`, :meth:`.Rolling.min`, and :meth:`.Expanding.min` now support ``Numba`` execution with the ``engine`` keyword (:issue:`38895`, :issue:`41267`)
 - :meth:`DataFrame.apply` can now accept NumPy unary operators as strings, e.g. ``df.apply("sqrt")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
@@ -333,6 +334,31 @@ values as measured by ``np.allclose``. Now no such casting occurs.
 
     df.groupby('key').agg(lambda x: x.sum())
 
+``float`` result for :meth:`.GroupBy.mean`, :meth:`.GroupBy.median`, and :meth:`.GroupBy.var`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Previously, these methods could result in different dtypes depending on the input values.
+Now, these methods will always return a float dtype. (:issue:`41137`)
+
+.. ipython:: python
+
+    df = pd.DataFrame({'a': [True], 'b': [1], 'c': [1.0]})
+
+*pandas 1.2.x*
+
+.. code-block:: ipython
+
+    In [5]: df.groupby(df.index).mean()
+    Out[5]:
+            a  b    c
+    0    True  1  1.0
+
+*pandas 1.3.0*
+
+.. ipython:: python
+
+    df.groupby(df.index).mean()
+
 Try operating inplace when setting values with ``loc`` and ``iloc``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -547,7 +573,7 @@ If installed, we now require:
 +-----------------+-----------------+----------+---------+
 | bottleneck      | 1.2.1           |          |         |
 +-----------------+-----------------+----------+---------+
-| numexpr         | 2.6.8           |          |         |
+| numexpr         | 2.7.0           |          |    X    |
 +-----------------+-----------------+----------+---------+
 | pytest (dev)    | 6.0             |          |    X    |
 +-----------------+-----------------+----------+---------+
@@ -648,9 +674,52 @@ Deprecations
 - Deprecated setting :attr:`Categorical._codes`, create a new :class:`Categorical` with the desired codes instead (:issue:`40606`)
 - Deprecated behavior of :meth:`DatetimeIndex.union` with mixed timezones; in a future version both will be cast to UTC instead of object dtype (:issue:`39328`)
 - Deprecated using ``usecols`` with out of bounds indices for ``read_csv`` with ``engine="c"`` (:issue:`25623`)
-- Deprecated passing arguments as positional in (:issue:`41485`) :
-  - :meth:`DataFrame.interpolate` (other than ``"method"``) and :meth:`Series.interpolate`
-  - :meth:`DataFrame.reset_index` (other than ``"level"``) and :meth:`Series.reset_index`
+- Deprecated special treatment of lists with first element a Categorical in the :class:`DataFrame` constructor; pass as ``pd.DataFrame({col: categorical, ...})`` instead (:issue:`38845`)
+- Deprecated passing arguments as positional (except for ``"method"``) in :meth:`DataFrame.interpolate` and :meth:`Series.interpolate` (:issue:`41485`)
+- Deprecated passing arguments (apart from ``value``) as positional in :meth:`DataFrame.fillna` and :meth:`Series.fillna` (:issue:`41485`)
+- Deprecated passing arguments as positional in :meth:`DataFrame.reset_index` (other than ``"level"``) and :meth:`Series.reset_index` (:issue:`41485`)
+- Deprecated construction of :class:`Series` or :class:`DataFrame` with ``DatetimeTZDtype`` data and ``datetime64[ns]`` dtype.  Use ``Series(data).dt.tz_localize(None)`` instead (:issue:`41555`,:issue:`33401`)
+
+.. _whatsnew_130.deprecations.nuisance_columns:
+
+Deprecated Dropping Nuisance Columns in DataFrame Reductions and DataFrameGroupBy Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The default of calling a reduction (.min, .max, .sum, ...) on a :class:`DataFrame` with
+``numeric_only=None`` (the default, columns on which the reduction raises ``TypeError``
+are silently ignored and dropped from the result.
+
+This behavior is deprecated. In a future version, the ``TypeError`` will be raised,
+and users will need to select only valid columns before calling the function.
+
+For example:
+
+.. ipython:: python
+
+   df = pd.DataFrame({"A": [1, 2, 3, 4], "B": pd.date_range("2016-01-01", periods=4)})
+   df
+
+*Old behavior*:
+
+.. code-block:: ipython
+
+    In [3]: df.prod()
+    Out[3]:
+    Out[3]:
+    A    24
+    dtype: int64
+
+*Future behavior*:
+
+.. code-block:: ipython
+
+    In [4]: df.prod()
+    ...
+    TypeError: 'DatetimeArray' does not implement reduction 'prod'
+
+    In [5]: df[["A"]].prod()
+    Out[5]:
+    A    24
+    dtype: int64
 
 .. ---------------------------------------------------------------------------
 
@@ -751,6 +820,8 @@ Conversion
 - Bug in :func:`factorize` where, when given an array with a numeric numpy dtype lower than int64, uint64 and float64, the unique values did not keep their original dtype (:issue:`41132`)
 - Bug in :class:`DataFrame` construction with a dictionary containing an arraylike with ``ExtensionDtype`` and ``copy=True`` failing to make a copy (:issue:`38939`)
 - Bug in :meth:`qcut` raising error when taking ``Float64DType`` as input (:issue:`40730`)
+- Bug in :class:`DataFrame` and :class:`Series` construction with ``datetime64[ns]`` data and ``dtype=object`` resulting in ``datetime`` objects instead of :class:`Timestamp` objects (:issue:`41599`)
+- Bug in :class:`DataFrame` and :class:`Series` construction with ``timedelta64[ns]`` data and ``dtype=object`` resulting in ``np.timedelta64`` objects instead of :class:`Timedelta` objects (:issue:`41599`)
 
 Strings
 ^^^^^^^
@@ -810,6 +881,7 @@ Missing
 - Bug in :func:`isna`, and :meth:`Series.isna`, :meth:`Index.isna`, :meth:`DataFrame.isna` (and the corresponding ``notna`` functions) not recognizing ``Decimal("NaN")`` objects (:issue:`39409`)
 - Bug in :meth:`DataFrame.fillna` not accepting dictionary for ``downcast`` keyword (:issue:`40809`)
 - Bug in :func:`isna` not returning a copy of the mask for nullable types, causing any subsequent mask modification to change the original array (:issue:`40935`)
+- Bug in :class:`DataFrame` construction with float data containing ``NaN`` and an integer ``dtype`` casting instead of retaining the ``NaN`` (:issue:`26919`)
 
 MultiIndex
 ^^^^^^^^^^
@@ -846,12 +918,15 @@ I/O
 - Bug in :func:`read_excel` dropping empty values from single-column spreadsheets (:issue:`39808`)
 - Bug in :func:`read_excel` loading trailing empty rows/columns for some filetypes (:issue:`41167`)
 - Bug in :func:`read_excel` raising ``AttributeError`` with ``MultiIndex`` header followed by two empty rows and no index, and bug affecting :func:`read_excel`, :func:`read_csv`, :func:`read_table`, :func:`read_fwf`, and :func:`read_clipboard` where one blank row after a ``MultiIndex`` header with no index would be dropped (:issue:`40442`)
-- Bug in :meth:`DataFrame.to_string` misplacing the truncation column when ``index=False`` (:issue:`40907`)
+- Bug in :meth:`DataFrame.to_string` misplacing the truncation column when ``index=False`` (:issue:`40904`)
+- Bug in :meth:`DataFrame.to_string` adding an extra dot and misaligning the truncation row when ``index=False`` (:issue:`40904`)
 - Bug in :func:`read_orc` always raising ``AttributeError`` (:issue:`40918`)
 - Bug in :func:`read_csv` and :func:`read_table` silently ignoring ``prefix`` if ``names`` and ``prefix`` are defined, now raising ``ValueError`` (:issue:`39123`)
 - Bug in :func:`read_csv` and :func:`read_excel` not respecting dtype for duplicated column name when ``mangle_dupe_cols`` is set to ``True`` (:issue:`35211`)
 - Bug in :func:`read_csv` and :func:`read_table` misinterpreting arguments when ``sys.setprofile`` had been previously called (:issue:`41069`)
 - Bug in the conversion from pyarrow to pandas (e.g. for reading Parquet) with nullable dtypes and a pyarrow array whose data buffer size is not a multiple of dtype size (:issue:`40896`)
+- Bug in :func:`read_excel` would raise an error when pandas could not determine the file type, even when user specified the ``engine`` argument (:issue:`41225`)
+-
 
 Period
 ^^^^^^
@@ -975,6 +1050,7 @@ Other
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
 - Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)
+- Bug in :meth:`DataFrame.agg()` not sorting the aggregated axis in the order of the provided aggragation functions when one or more aggregation function fails to produce results (:issue:`33634`)
 - Bug in :meth:`DataFrame.clip` not interpreting missing values as no threshold (:issue:`40420`)
 - Bug in :class:`Series` backed by :class:`DatetimeArray` or :class:`TimedeltaArray` sometimes failing to set the array's ``freq`` to ``None`` (:issue:`41425`)
 

environment.yml

@@ -81,7 +81,7 @@ dependencies:
   - ipython>=7.11.1
   - jinja2<3.0.0  # pandas.Styler
   - matplotlib>=2.2.2  # pandas.plotting, Series.plot, DataFrame.plot
-  - numexpr>=2.6.8
+  - numexpr>=2.7.0
   - scipy>=1.2
   - numba>=0.46.0
 

pandas/_libs/reduction.pyx

@@ -1,4 +1,3 @@
-from copy import copy
 
 from libc.stdlib cimport (
     free,
@@ -307,13 +306,11 @@ cpdef inline extract_result(object res):
         # Preserve EA
         res = res._values
         if res.ndim == 1 and len(res) == 1:
+            # see test_agg_lambda_with_timezone, test_resampler_grouper.py::test_apply
             res = res[0]
-    if hasattr(res, 'values') and is_array(res.values):
-        res = res.values
     if is_array(res):
-        if res.ndim == 0:
-            res = res.item()
-        elif res.ndim == 1 and len(res) == 1:
+        if res.ndim == 1 and len(res) == 1:
+            # see test_resampler_grouper.py::test_apply
             res = res[0]
     return res
 
@@ -386,7 +383,7 @@ def apply_frame_axis0(object frame, object f, object names,
             # Need to infer if low level index slider will cause segfaults
             require_slow_apply = i == 0 and piece is chunk
             try:
-                if not piece.index is chunk.index:
+                if piece.index is not chunk.index:
                     mutated = True
             except AttributeError:
                 # `piece` might not have an index, could be e.g. an int
@@ -397,7 +394,7 @@ def apply_frame_axis0(object frame, object f, object names,
                 try:
                     piece = piece.copy(deep="all")
                 except (TypeError, AttributeError):
-                    piece = copy(piece)
+                    pass
 
             results.append(piece)
 

pandas/compat/_optional.py

@@ -17,7 +17,7 @@
     "gcsfs": "0.6.0",
     "lxml.etree": "4.3.0",
     "matplotlib": "2.2.3",
-    "numexpr": "2.6.8",
+    "numexpr": "2.7.0",
     "odfpy": "1.3.0",
     "openpyxl": "3.0.0",
     "pandas_gbq": "0.12.0",

pandas/core/apply.py

@@ -376,12 +376,10 @@ def agg_list_like(self) -> FrameOrSeriesUnion:
             raise ValueError("no results")
 
         try:
-            return concat(results, keys=keys, axis=1, sort=False)
+            concatenated = concat(results, keys=keys, axis=1, sort=False)
         except TypeError as err:
-
             # we are concatting non-NDFrame objects,
             # e.g. a list of scalars
-
             from pandas import Series
 
             result = Series(results, index=keys, name=obj.name)
@@ -390,6 +388,16 @@ def agg_list_like(self) -> FrameOrSeriesUnion:
                     "cannot combine transform and aggregation operations"
                 ) from err
             return result
+        else:
+            # Concat uses the first index to determine the final indexing order.
+            # The union of a shorter first index with the other indices causes
+            # the index sorting to be different from the order of the aggregating
+            # functions. Reindex if this is the case.
+            index_size = concatenated.index.size
+            full_ordered_index = next(
+                result.index for result in results if result.index.size == index_size
+            )
+            return concatenated.reindex(full_ordered_index, copy=False)
 
     def agg_dict_like(self) -> FrameOrSeriesUnion:
         """

pandas/core/arrays/base.py

@@ -493,8 +493,7 @@ def size(self) -> int:
         """
         The number of elements in the array.
         """
-        # error: Incompatible return value type (got "number", expected "int")
-        return np.prod(self.shape)  # type: ignore[return-value]
+        return np.prod(self.shape)
 
     @property
     def ndim(self) -> int:

pandas/core/arrays/sparse/accessor.py

@@ -354,9 +354,8 @@ def density(self) -> float:
         """
         Ratio of non-sparse points to total (dense) data points.
         """
-        # error: Incompatible return value type (got "number", expected "float")
         tmp = np.mean([column.array.density for _, column in self._parent.items()])
-        return tmp  # type: ignore[return-value]
+        return tmp
 
     @staticmethod
     def _prep_index(data, index, columns):

pandas/core/computation/ops.py

@@ -27,7 +27,6 @@
     result_type_many,
 )
 from pandas.core.computation.scope import DEFAULT_GLOBALS
-from pandas.util.version import Version
 
 from pandas.io.formats.printing import (
     pprint_thing,
@@ -616,18 +615,8 @@ def __repr__(self) -> str:
 
 class FuncNode:
     def __init__(self, name: str):
-        from pandas.core.computation.check import (
-            NUMEXPR_INSTALLED,
-            NUMEXPR_VERSION,
-        )
-
-        if name not in MATHOPS or (
-            NUMEXPR_INSTALLED
-            and Version(NUMEXPR_VERSION) < Version("2.6.9")
-            and name in ("floor", "ceil")
-        ):
+        if name not in MATHOPS:
             raise ValueError(f'"{name}" is not a supported function')
-
         self.name = name
         self.func = getattr(np, name)
 

pandas/core/config_init.py

@@ -726,3 +726,26 @@ def register_converter_cb(key):
         validator=is_one_of_factory(["auto", True, False]),
         cb=register_converter_cb,
     )
+
+# ------
+# Styler
+# ------
+
+styler_sparse_index_doc = """
+: bool
+    Whether to sparsify the display of a hierarchical index. Setting to False will
+    display each explicit level element in a hierarchical key for each row.
+"""
+
+styler_sparse_columns_doc = """
+: bool
+    Whether to sparsify the display of hierarchical columns. Setting to False will
+    display each explicit level element in a hierarchical key for each column.
+"""
+
+with cf.config_prefix("styler"):
+    cf.register_option("sparse.index", True, styler_sparse_index_doc, validator=bool)
+
+    cf.register_option(
+        "sparse.columns", True, styler_sparse_columns_doc, validator=bool
+    )