Merge branch 'master' into 42916

Author: debnathshoham

.github/workflows/database.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
@@ -79,7 +80,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: flexible
+        channel-priority: strict
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

asv_bench/benchmarks/algorithms.py

@@ -44,9 +44,9 @@ def setup(self, unique, sort, dtype):
                 raise NotImplementedError
 
         data = {
-            "int": pd.Int64Index(np.arange(N)),
-            "uint": pd.UInt64Index(np.arange(N)),
-            "float": pd.Float64Index(np.random.randn(N)),
+            "int": pd.Index(np.arange(N), dtype="int64"),
+            "uint": pd.Index(np.arange(N), dtype="uint64"),
+            "float": pd.Index(np.random.randn(N), dtype="float64"),
             "object": string_index,
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(
@@ -76,9 +76,9 @@ class Duplicated:
     def setup(self, unique, keep, dtype):
         N = 10 ** 5
         data = {
-            "int": pd.Int64Index(np.arange(N)),
-            "uint": pd.UInt64Index(np.arange(N)),
-            "float": pd.Float64Index(np.random.randn(N)),
+            "int": pd.Index(np.arange(N), dtype="int64"),
+            "uint": pd.Index(np.arange(N), dtype="uint64"),
+            "float": pd.Index(np.random.randn(N), dtype="float64"),
             "string": tm.makeStringIndex(N),
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(

asv_bench/benchmarks/groupby.py

@@ -603,6 +603,38 @@ def time_sum(self):
         self.df.groupby(["a"])["b"].sum()
 
 
+class String:
+    # GH#41596
+    param_names = ["dtype", "method"]
+    params = [
+        ["str", "string[python]"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
+    ]
+
+    def setup(self, dtype, method):
+        cols = list("abcdefghjkl")
+        self.df = DataFrame(
+            np.random.randint(0, 100, size=(1_000_000, len(cols))),
+            columns=cols,
+            dtype=dtype,
+        )
+
+    def time_str_func(self, dtype, method):
+        self.df.groupby("a")[self.df.columns[1:]].agg(method)
+
+
 class Categories:
     def setup(self):
         N = 10 ** 5

asv_bench/benchmarks/indexing_engines.py

@@ -48,7 +48,7 @@ def setup(self, engine_and_dtype, index_type):
             "non_monotonic": np.array([1, 2, 3] * N, dtype=dtype),
         }[index_type]
 
-        self.data = engine(lambda: arr, len(arr))
+        self.data = engine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc(2)
 
@@ -70,7 +70,7 @@ def setup(self, index_type):
             "non_monotonic": np.array(list("abc") * N, dtype=object),
         }[index_type]
 
-        self.data = libindex.ObjectEngine(lambda: arr, len(arr))
+        self.data = libindex.ObjectEngine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc("b")
 

asv_bench/benchmarks/io/style.py

@@ -42,6 +42,14 @@ def peakmem_format_render(self, cols, rows):
         self._style_format()
         self.st._render_html(True, True)
 
+    def time_apply_format_hide_render(self, cols, rows):
+        self._style_apply_format_hide()
+        self.st._render_html(True, True)
+
+    def peakmem_apply_format_hide_render(self, cols, rows):
+        self._style_apply_format_hide()
+        self.st._render_html(True, True)
+
     def _style_apply(self):
         def _apply_func(s):
             return [
@@ -63,3 +71,9 @@ def _style_format(self):
         self.st = self.df.style.format(
             "{:,.3f}", subset=IndexSlice["row_1":f"row_{ir}", "float_1":f"float_{ic}"]
         )
+
+    def _style_apply_format_hide(self):
+        self.st = self.df.style.applymap(lambda v: "color: red;")
+        self.st.format("{:.3f}")
+        self.st.hide_index(self.st.index[1:])
+        self.st.hide_columns(self.st.columns[1:])

asv_bench/benchmarks/sparse.py

@@ -180,4 +180,19 @@ def time_min_max(self, func, fill_value):
         getattr(self.sp_arr, func)()
 
 
+class Take:
+
+    params = ([np.array([0]), np.arange(100_000), np.full(100_000, -1)], [True, False])
+    param_names = ["indices", "allow_fill"]
+
+    def setup(self, indices, allow_fill):
+        N = 1_000_000
+        fill_value = 0.0
+        arr = make_array(N, 1e-5, fill_value, np.float64)
+        self.sp_arr = SparseArray(arr, fill_value=fill_value)
+
+    def time_take(self, indices, allow_fill):
+        self.sp_arr.take(indices, allow_fill=allow_fill)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

doc/source/whatsnew/v1.3.4.rst

@@ -17,9 +17,11 @@ Fixed regressions
 - Fixed regression in :meth:`merge` with integer and ``NaN`` keys failing with ``outer`` merge (:issue:`43550`)
 - Fixed regression in :meth:`DataFrame.corr` raising ``ValueError`` with ``method="spearman"`` on 32-bit platforms (:issue:`43588`)
 - Fixed performance regression in :meth:`MultiIndex.equals` (:issue:`43549`)
+- Fixed performance regression in :meth:`.GroupBy.first` and :meth:`.GroupBy.last` with :class:`StringDtype` (:issue:`41596`)
 - Fixed regression in :meth:`Series.cat.reorder_categories` failing to update the categories on the ``Series`` (:issue:`43232`)
 - Fixed regression in :meth:`Series.cat.categories` setter failing to update the categories on the ``Series`` (:issue:`43334`)
--
+- Fixed regression in :meth:`pandas.read_csv` raising ``UnicodeDecodeError`` exception when ``memory_map=True`` (:issue:`43540`)
+- Fixed regression in :meth:`Series.aggregate` attempting to pass ``args`` and ``kwargs`` multiple times to the user supplied ``func`` in certain cases (:issue:`43357`)
 
 .. ---------------------------------------------------------------------------
 
@@ -28,6 +30,7 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 - Fixed bug in :meth:`.GroupBy.mean` with datetimelike values including ``NaT`` values returning incorrect results (:issue:`43132`)
+- Fixed bug in :meth:`Series.aggregate` not passing the first ``args`` to the user supplied ``func`` in certain cases (:issue:`43357`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.4.0.rst

@@ -79,6 +79,7 @@ Styler
   - Keyword arguments ``level`` and ``names`` added to :meth:`.Styler.hide_index` and :meth:`.Styler.hide_columns` for additional control of visibility of MultiIndexes and index names (:issue:`25475`, :issue:`43404`, :issue:`43346`)
   - Global options have been extended to configure default ``Styler`` properties including formatting and encoding and mathjax options and LaTeX (:issue:`41395`)
   - Naive sparsification is now possible for LaTeX without the multirow package (:issue:`43369`)
+  - :meth:`Styler.to_html` omits CSSStyle rules for hidden table elements (:issue:`43619`)
 
 Formerly Styler relied on ``display.html.use_mathjax``, which has now been replaced by ``styler.html.mathjax``.
 
@@ -123,6 +124,7 @@ Other enhancements
 - Methods that relied on hashmap based algos such as :meth:`DataFrameGroupBy.value_counts`, :meth:`DataFrameGroupBy.count` and :func:`factorize` ignored imaginary component for complex numbers (:issue:`17927`)
 - Add :meth:`Series.str.removeprefix` and :meth:`Series.str.removesuffix` introduced in Python 3.9 to remove pre-/suffixes from string-type :class:`Series` (:issue:`36944`)
 - Attempting to write into a file in missing parent directory with :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_html`, :meth:`DataFrame.to_excel`, :meth:`DataFrame.to_feather`, :meth:`DataFrame.to_parquet`, :meth:`DataFrame.to_stata`, :meth:`DataFrame.to_json`, :meth:`DataFrame.to_pickle`, and :meth:`DataFrame.to_xml` now explicitly mentions missing parent directory, the same is true for :class:`Series` counterparts (:issue:`24306`)
+- :meth:`IntegerArray.all` , :meth:`IntegerArray.any`, :meth:`FloatingArray.any`, and :meth:`FloatingArray.all` use Kleene logic (:issue:`41967`)
 - Added support for nullable boolean and integer types in :meth:`DataFrame.to_stata`, :class:`~pandas.io.stata.StataWriter`, :class:`~pandas.io.stata.StataWriter117`, and :class:`~pandas.io.stata.StataWriterUTF8` (:issue:`40855`)
 -
 
@@ -334,7 +336,7 @@ Other Deprecations
 - Deprecated the 'include_start' and 'include_end' arguments in :meth:`DataFrame.between_time`; in a future version passing 'include_start' or 'include_end' will raise (:issue:`40245`)
 - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`)
 - Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`)
--
+- Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`)
 
 .. ---------------------------------------------------------------------------
 
@@ -354,7 +356,8 @@ Performance improvements
 - Performance improvement in indexing with a :class:`MultiIndex` indexer on another :class:`MultiIndex` (:issue:43370`)
 - Performance improvement in :meth:`GroupBy.quantile` (:issue:`43469`)
 - :meth:`SparseArray.min` and :meth:`SparseArray.max` no longer require converting to a dense array (:issue:`43526`)
--
+- Performance improvement in :meth:`SparseArray.take` with ``allow_fill=False`` (:issue:`43654`)
+- Performance improvement in :meth:`.Rolling.mean` and :meth:`.Expanding.mean` with ``engine="numba"`` (:issue:`43612`)
 
 .. ---------------------------------------------------------------------------
 
@@ -486,6 +489,7 @@ Reshaping
 - Bug in :meth:`DataFrame.append` failing to retain dtypes when appended columns do not match (:issue:`43392`)
 - Bug in :func:`concat` of ``bool`` and ``boolean`` dtypes resulting in ``object`` dtype instead of ``boolean`` dtype (:issue:`42800`)
 - Bug in :func:`crosstab` when inputs are are categorical Series, there are categories that are not present in one or both of the Series, and ``margins=True``. Previously the margin value for missing categories was ``NaN``. It is now correctly reported as 0 (:issue:`43505`)
+- Bug in :func:`concat` would fail when the ``objs`` argument all had the same index and the ``keys`` argument contained duplicates (:issue:`43595`)
 
 Sparse
 ^^^^^^

pandas/_libs/index.pyi

@@ -6,7 +6,7 @@ from pandas import MultiIndex
 
 class IndexEngine:
     over_size_threshold: bool
-    def __init__(self, vgetter, n: int): ...
+    def __init__(self, values: np.ndarray): ...
     def __contains__(self, val: object) -> bool: ...
     # -> int | slice | np.ndarray[bool]
     def get_loc(self, val: object) -> int | slice | np.ndarray: ...