Merge branch 'master' into styler_bar_colors

Author: attack68

.github/workflows/database.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
@@ -79,7 +80,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: flexible
+        channel-priority: strict
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

asv_bench/benchmarks/algorithms.py

@@ -44,9 +44,9 @@ def setup(self, unique, sort, dtype):
                 raise NotImplementedError
 
         data = {
-            "int": pd.Int64Index(np.arange(N)),
-            "uint": pd.UInt64Index(np.arange(N)),
-            "float": pd.Float64Index(np.random.randn(N)),
+            "int": pd.Index(np.arange(N), dtype="int64"),
+            "uint": pd.Index(np.arange(N), dtype="uint64"),
+            "float": pd.Index(np.random.randn(N), dtype="float64"),
             "object": string_index,
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(
@@ -76,9 +76,9 @@ class Duplicated:
     def setup(self, unique, keep, dtype):
         N = 10 ** 5
         data = {
-            "int": pd.Int64Index(np.arange(N)),
-            "uint": pd.UInt64Index(np.arange(N)),
-            "float": pd.Float64Index(np.random.randn(N)),
+            "int": pd.Index(np.arange(N), dtype="int64"),
+            "uint": pd.Index(np.arange(N), dtype="uint64"),
+            "float": pd.Index(np.random.randn(N), dtype="float64"),
             "string": tm.makeStringIndex(N),
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(

asv_bench/benchmarks/io/style.py

@@ -42,6 +42,14 @@ def peakmem_format_render(self, cols, rows):
         self._style_format()
         self.st._render_html(True, True)
 
+    def time_apply_format_hide_render(self, cols, rows):
+        self._style_apply_format_hide()
+        self.st._render_html(True, True)
+
+    def peakmem_apply_format_hide_render(self, cols, rows):
+        self._style_apply_format_hide()
+        self.st._render_html(True, True)
+
     def _style_apply(self):
         def _apply_func(s):
             return [
@@ -63,3 +71,9 @@ def _style_format(self):
         self.st = self.df.style.format(
             "{:,.3f}", subset=IndexSlice["row_1":f"row_{ir}", "float_1":f"float_{ic}"]
         )
+
+    def _style_apply_format_hide(self):
+        self.st = self.df.style.applymap(lambda v: "color: red;")
+        self.st.format("{:.3f}")
+        self.st.hide_index(self.st.index[1:])
+        self.st.hide_columns(self.st.columns[1:])

asv_bench/benchmarks/sparse.py

@@ -180,4 +180,19 @@ def time_min_max(self, func, fill_value):
         getattr(self.sp_arr, func)()
 
 
+class Take:
+
+    params = ([np.array([0]), np.arange(100_000), np.full(100_000, -1)], [True, False])
+    param_names = ["indices", "allow_fill"]
+
+    def setup(self, indices, allow_fill):
+        N = 1_000_000
+        fill_value = 0.0
+        arr = make_array(N, 1e-5, fill_value, np.float64)
+        self.sp_arr = SparseArray(arr, fill_value=fill_value)
+
+    def time_take(self, indices, allow_fill):
+        self.sp_arr.take(indices, allow_fill=allow_fill)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

doc/source/whatsnew/v1.3.4.rst

@@ -19,6 +19,7 @@ Fixed regressions
 - Fixed performance regression in :meth:`MultiIndex.equals` (:issue:`43549`)
 - Fixed regression in :meth:`Series.cat.reorder_categories` failing to update the categories on the ``Series`` (:issue:`43232`)
 - Fixed regression in :meth:`Series.cat.categories` setter failing to update the categories on the ``Series`` (:issue:`43334`)
+- Fixed regression in :meth:`pandas.read_csv` raising ``UnicodeDecodeError`` exception when ``memory_map=True`` (:issue:`43540`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.4.0.rst

@@ -79,6 +79,7 @@ Styler
   - Keyword arguments ``level`` and ``names`` added to :meth:`.Styler.hide_index` and :meth:`.Styler.hide_columns` for additional control of visibility of MultiIndexes and index names (:issue:`25475`, :issue:`43404`, :issue:`43346`)
   - Global options have been extended to configure default ``Styler`` properties including formatting and encoding and mathjax options and LaTeX (:issue:`41395`)
   - Naive sparsification is now possible for LaTeX without the multirow package (:issue:`43369`)
+  - :meth:`Styler.to_html` omits CSSStyle rules for hidden table elements (:issue:`43619`)
 
 Formerly Styler relied on ``display.html.use_mathjax``, which has now been replaced by ``styler.html.mathjax``.
 
@@ -123,6 +124,7 @@ Other enhancements
 - Methods that relied on hashmap based algos such as :meth:`DataFrameGroupBy.value_counts`, :meth:`DataFrameGroupBy.count` and :func:`factorize` ignored imaginary component for complex numbers (:issue:`17927`)
 - Add :meth:`Series.str.removeprefix` and :meth:`Series.str.removesuffix` introduced in Python 3.9 to remove pre-/suffixes from string-type :class:`Series` (:issue:`36944`)
 - Attempting to write into a file in missing parent directory with :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_html`, :meth:`DataFrame.to_excel`, :meth:`DataFrame.to_feather`, :meth:`DataFrame.to_parquet`, :meth:`DataFrame.to_stata`, :meth:`DataFrame.to_json`, :meth:`DataFrame.to_pickle`, and :meth:`DataFrame.to_xml` now explicitly mentions missing parent directory, the same is true for :class:`Series` counterparts (:issue:`24306`)
+- :meth:`IntegerArray.all` , :meth:`IntegerArray.any`, :meth:`FloatingArray.any`, and :meth:`FloatingArray.all` use Kleene logic (:issue:`41967`)
 - Added support for nullable boolean and integer types in :meth:`DataFrame.to_stata`, :class:`~pandas.io.stata.StataWriter`, :class:`~pandas.io.stata.StataWriter117`, and :class:`~pandas.io.stata.StataWriterUTF8` (:issue:`40855`)
 -
 
@@ -354,6 +356,7 @@ Performance improvements
 - Performance improvement in indexing with a :class:`MultiIndex` indexer on another :class:`MultiIndex` (:issue:43370`)
 - Performance improvement in :meth:`GroupBy.quantile` (:issue:`43469`)
 - :meth:`SparseArray.min` and :meth:`SparseArray.max` no longer require converting to a dense array (:issue:`43526`)
+- Performance improvement in :meth:`SparseArray.take` with ``allow_fill=False`` (:issue:`43654`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/_libs/index.pyx

@@ -84,7 +84,7 @@ cdef class IndexEngine:
         if self.over_size_threshold and self.is_monotonic_increasing:
             if not self.is_unique:
                 return self._get_loc_duplicates(val)
-            values = self._get_index_values()
+            values = self.values
 
             self._check_type(val)
             try:
@@ -116,7 +116,7 @@ cdef class IndexEngine:
             Py_ssize_t diff
 
         if self.is_monotonic_increasing:
-            values = self._get_index_values()
+            values = self.values
             try:
                 left = values.searchsorted(val, side='left')
                 right = values.searchsorted(val, side='right')
@@ -139,7 +139,7 @@ cdef class IndexEngine:
         cdef:
             ndarray[uint8_t, ndim=1, cast=True] indexer
 
-        indexer = self._get_index_values() == val
+        indexer = self.values == val
         return self._unpack_bool_indexer(indexer, val)
 
     cdef _unpack_bool_indexer(self,
@@ -199,7 +199,7 @@ cdef class IndexEngine:
         cdef:
             bint is_unique
         try:
-            values = self._get_index_values()
+            values = self.values
             self.monotonic_inc, self.monotonic_dec, is_unique = \
                 self._call_monotonic(values)
         except TypeError:
@@ -214,17 +214,14 @@ cdef class IndexEngine:
             self.unique = 1
             self.need_unique_check = 0
 
-    cdef ndarray _get_index_values(self):
-        return self.values
-
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=False)
 
     def get_backfill_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
-        return algos.backfill(self._get_index_values(), other, limit=limit)
+        return algos.backfill(self.values, other, limit=limit)
 
     def get_pad_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
-        return algos.pad(self._get_index_values(), other, limit=limit)
+        return algos.pad(self.values, other, limit=limit)
 
     cdef _make_hash_table(self, Py_ssize_t n):
         raise NotImplementedError
@@ -243,7 +240,7 @@ cdef class IndexEngine:
 
         if not self.is_mapping_populated:
 
-            values = self._get_index_values()
+            values = self.values
             self.mapping = self._make_hash_table(len(values))
             self._call_map_locations(values)
 
@@ -291,7 +288,7 @@ cdef class IndexEngine:
             bint d_has_nan = False, stargets_has_nan = False, need_nan_check = True
 
         self._ensure_mapping_populated()
-        values = np.array(self._get_index_values(), copy=False)
+        values = self.values
         stargets = set(targets)
 
         n = len(values)
@@ -411,9 +408,6 @@ cdef class ObjectEngine(IndexEngine):
 
 cdef class DatetimeEngine(Int64Engine):
 
-    cdef str _get_box_dtype(self):
-        return 'M8[ns]'
-
     cdef int64_t _unbox_scalar(self, scalar) except? -1:
         # NB: caller is responsible for ensuring tzawareness compat
         #  before we get here
@@ -431,16 +425,13 @@ cdef class DatetimeEngine(Int64Engine):
         if self.over_size_threshold and self.is_monotonic_increasing:
             if not self.is_unique:
                 return self._get_loc_duplicates(conv)
-            values = self._get_index_values()
+            values = self.values
             loc = values.searchsorted(conv, side='left')
             return values[loc] == conv
 
         self._ensure_mapping_populated()
         return conv in self.mapping
 
-    cdef ndarray _get_index_values(self):
-        return self.values.view('i8')
-
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=True)
 
@@ -462,7 +453,7 @@ cdef class DatetimeEngine(Int64Engine):
         if self.over_size_threshold and self.is_monotonic_increasing:
             if not self.is_unique:
                 return self._get_loc_duplicates(conv)
-            values = self._get_index_values()
+            values = self.values
 
             loc = values.searchsorted(conv, side='left')
 
@@ -479,35 +470,9 @@ cdef class DatetimeEngine(Int64Engine):
         except KeyError:
             raise KeyError(val)
 
-    def get_indexer_non_unique(self, ndarray targets):
-        # we may get datetime64[ns] or timedelta64[ns], cast these to int64
-        return super().get_indexer_non_unique(targets.view("i8"))
-
-    def get_indexer(self, ndarray values) -> np.ndarray:
-        self._ensure_mapping_populated()
-        if values.dtype != self._get_box_dtype():
-            return np.repeat(-1, len(values)).astype(np.intp)
-        values = np.asarray(values).view('i8')
-        return self.mapping.lookup(values)
-
-    def get_pad_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
-        if other.dtype != self._get_box_dtype():
-            return np.repeat(-1, len(other)).astype(np.intp)
-        other = np.asarray(other).view('i8')
-        return algos.pad(self._get_index_values(), other, limit=limit)
-
-    def get_backfill_indexer(self, other: np.ndarray, limit=None) -> np.ndarray:
-        if other.dtype != self._get_box_dtype():
-            return np.repeat(-1, len(other)).astype(np.intp)
-        other = np.asarray(other).view('i8')
-        return algos.backfill(self._get_index_values(), other, limit=limit)
-
 
 cdef class TimedeltaEngine(DatetimeEngine):
 
-    cdef str _get_box_dtype(self):
-        return 'm8[ns]'
-
     cdef int64_t _unbox_scalar(self, scalar) except? -1:
         if not (isinstance(scalar, _Timedelta) or scalar is NaT):
             raise TypeError(scalar)

pandas/_libs/index_class_helper.pxi.in

@@ -49,13 +49,11 @@ cdef class {{name}}Engine(IndexEngine):
         # Returns ndarray[bool] or int
         cdef:
             ndarray[uint8_t, ndim=1, cast=True] indexer
-            ndarray[intp_t, ndim=1] found
             ndarray[{{dtype}}_t, ndim=1] values
-            int count = 0
 
         self._check_type(val)
 
-        values = self._get_index_values()
+        values = self.values
         try:
             with warnings.catch_warnings():
                 # e.g. if values is float64 and `val` is a str, suppress warning

pandas/core/algorithms.py

@@ -422,7 +422,7 @@ def unique(values):
     """
     values = _ensure_arraylike(values)
 
-    if is_extension_array_dtype(values):
+    if is_extension_array_dtype(values.dtype):
         # Dispatch to extension dtype's unique.
         return values.unique()