Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

ci/deps/actions-39-slow.yaml

@@ -14,25 +14,29 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - fsspec>=0.7.4, <2021.6.0
+  - bottleneck
+  - fsspec>=0.8.0, <2021.6.0
+  - gcsfs
   - html5lib
+  - jinja2
   - lxml
   - matplotlib
+  - moto>=1.3.14
+  - flask
   - numexpr
   - numpy
   - openpyxl
-  - patsy
-  - psycopg2
-  - pymysql
+  - pyarrow
   - pytables
   - python-dateutil
   - pytz
-  - s3fs>=0.4.0
-  - moto>=1.3.14
+  - s3fs>=0.4.2
   - scipy
   - sqlalchemy
   - xlrd
   - xlsxwriter
-  - moto
-  - flask
-  - numba
+  - xlwt
+  - pyreadstat
+  - pip
+  - pip:
+    - pyxlsb

ci/deps/actions-39.yaml

@@ -12,11 +12,30 @@ dependencies:
   - hypothesis>=5.5.3
 
   # pandas dependencies
+  - beautifulsoup4
+  - bottleneck
+  - fsspec>=0.8.0, <2021.6.0
+  - gcsfs
+  - html5lib
+  - jinja2
+  - lxml
+  - matplotlib
+  - moto>=1.3.14
+  - flask
+  - numexpr
   - numpy
+  - openpyxl
+  - pyarrow
+  - pytables
   - python-dateutil
   - pytz
-
-  # optional dependencies
-  - pytables
+  - s3fs>=0.4.2
   - scipy
-  - pyarrow=1.0
+  - sqlalchemy
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - pyreadstat
+  - pip
+  - pip:
+    - pyxlsb

doc/source/user_guide/style.ipynb

@@ -1190,9 +1190,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "In version 0.20.0 the ability to customize the bar chart further was given. You can now have the `df.style.bar` be centered on zero or midpoint value (in addition to the already existing way of having the min value at the left side of the cell), and you can pass a list of `[color_negative, color_positive]`.\n",
+    "Additional keyword arguments give more control on centering and positioning, and you can pass a list of `[color_negative, color_positive]` to highlight lower and higher values.\n",
     "\n",
-    "Here's how you can change the above with the new `align='mid'` option:"
+    "Here's how you can change the above with the new `align` option, combined with setting `vmin` and `vmax` limits, the `width` of the figure, and underlying css `props` of cells, leaving space to display the text and the bars:"
    ]
   },
   {
@@ -1201,7 +1201,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df2.style.bar(subset=['A', 'B'], align='mid', color=['#d65f5f', '#5fba7d'])"
+    "df2.style.bar(align=0, vmin=-2.5, vmax=2.5, color=['#d65f5f', '#5fba7d'],\n",
+    "              width=60, props=\"width: 120px; border-right: 1px solid black;\").format('{:.3f}', na_rep=\"\")"
    ]
   },
   {
@@ -1225,28 +1226,31 @@
     "\n",
     "# Test series\n",
     "test1 = pd.Series([-100,-60,-30,-20], name='All Negative')\n",
-    "test2 = pd.Series([10,20,50,100], name='All Positive')\n",
-    "test3 = pd.Series([-10,-5,0,90], name='Both Pos and Neg')\n",
+    "test2 = pd.Series([-10,-5,0,90], name='Both Pos and Neg')\n",
+    "test3 = pd.Series([10,20,50,100], name='All Positive')\n",
+    "test4 = pd.Series([100, 103, 101, 102], name='Large Positive')\n",
+    "\n",
     "\n",
     "head = \"\"\"\n",
     "<table>\n",
     "    <thead>\n",
     "        <th>Align</th>\n",
     "        <th>All Negative</th>\n",
-    "        <th>All Positive</th>\n",
     "        <th>Both Neg and Pos</th>\n",
+    "        <th>All Positive</th>\n",
+    "        <th>Large Positive</th>\n",
     "    </thead>\n",
     "    </tbody>\n",
     "\n",
     "\"\"\"\n",
     "\n",
-    "aligns = ['left','zero','mid']\n",
+    "aligns = ['left', 'right', 'zero', 'mid', 'mean', 99]\n",
     "for align in aligns:\n",
     "    row = \"<tr><th>{}</th>\".format(align)\n",
-    "    for series in [test1,test2,test3]:\n",
+    "    for series in [test1,test2,test3, test4]:\n",
     "        s = series.copy()\n",
     "        s.name=''\n",
-    "        row += \"<td>{}</td>\".format(s.to_frame().style.bar(align=align, \n",
+    "        row += \"<td>{}</td>\".format(s.to_frame().style.hide_index().bar(align=align, \n",
     "                                                           color=['#d65f5f', '#5fba7d'], \n",
     "                                                           width=100).render()) #testn['width']\n",
     "    row += '</tr>'\n",

doc/source/whatsnew/v1.3.1.rst

@@ -17,6 +17,7 @@ Fixed regressions
 - Pandas could not be built on PyPy (:issue:`42355`)
 - :class:`DataFrame` constructed with with an older version of pandas could not be unpickled (:issue:`42345`)
 - Performance regression in constructing a :class:`DataFrame` from a dictionary of dictionaries (:issue:`42338`)
+- Fixed regression in :meth:`DataFrame.agg` dropping values when the DataFrame had an Extension Array dtype, a duplicate index, and ``axis=1`` (:issue:`42380`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -25,7 +26,7 @@ Fixed regressions
 
 Bug fixes
 ~~~~~~~~~
--
+- Fixed bug in :meth:`DataFrame.transpose` dropping values when the DataFrame had an Extension Array dtype and a duplicate index (:issue:`42380`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.4.0.rst

@@ -30,6 +30,7 @@ enhancement2
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 - :meth:`Series.sample`, :meth:`DataFrame.sample`, and :meth:`.GroupBy.sample` now accept a ``np.random.Generator`` as input to ``random_state``. A generator will be more performant, especially with ``replace=False`` (:issue:`38100`)
+-  Additional options added to :meth:`.Styler.bar` to control alignment and display (:issue:`26070`)
 - :meth:`Series.ewm`, :meth:`DataFrame.ewm`, now support a ``method`` argument with a ``'table'`` option that performs the windowing operation over an entire :class:`DataFrame`. See :ref:`Window Overview <window.overview>` for performance and functional benefits (:issue:`42273`)
 -
 
@@ -160,7 +161,7 @@ Deprecations
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 - Performance improvement in :meth:`.GroupBy.sample`, especially when ``weights`` argument provided (:issue:`34483`)
--
+- Performance improvement in :meth:`.GroupBy.transform` for user-defined functions (:issue:`41598`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/frame.py

@@ -3342,8 +3342,8 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
             values = self.values
 
             new_values = [arr_type._from_sequence(row, dtype=dtype) for row in values]
-            result = self._constructor(
-                dict(zip(self.index, new_values)), index=self.columns
+            result = type(self)._from_arrays(
+                new_values, index=self.columns, columns=self.index
             )
 
         else:

pandas/core/groupby/generic.py

@@ -1308,42 +1308,35 @@ def _transform_general(self, func, *args, **kwargs):
         gen = self.grouper.get_iterator(obj, axis=self.axis)
         fast_path, slow_path = self._define_paths(func, *args, **kwargs)
 
-        for name, group in gen:
-            if group.size == 0:
-                continue
+        # Determine whether to use slow or fast path by evaluating on the first group.
+        # Need to handle the case of an empty generator and process the result so that
+        # it does not need to be computed again.
+        try:
+            name, group = next(gen)
+        except StopIteration:
+            pass
+        else:
             object.__setattr__(group, "name", name)
-
-            # Try slow path and fast path.
             try:
                 path, res = self._choose_path(fast_path, slow_path, group)
             except TypeError:
                 return self._transform_item_by_item(obj, fast_path)
             except ValueError as err:
                 msg = "transform must return a scalar value for each group"
                 raise ValueError(msg) from err
-
-            if isinstance(res, Series):
-
-                # we need to broadcast across the
-                # other dimension; this will preserve dtypes
-                # GH14457
-                if res.index.is_(obj.index):
-                    r = concat([res] * len(group.columns), axis=1)
-                    r.columns = group.columns
-                    r.index = group.index
-                else:
-                    r = self.obj._constructor(
-                        np.concatenate([res.values] * len(group.index)).reshape(
-                            group.shape
-                        ),
-                        columns=group.columns,
-                        index=group.index,
-                    )
-
-                applied.append(r)
-            else:
+            if group.size > 0:
+                res = _wrap_transform_general_frame(self.obj, group, res)
                 applied.append(res)
 
+        # Compute and process with the remaining groups
+        for name, group in gen:
+            if group.size == 0:
+                continue
+            object.__setattr__(group, "name", name)
+            res = path(group)
+            res = _wrap_transform_general_frame(self.obj, group, res)
+            applied.append(res)
+
         concat_index = obj.columns if self.axis == 0 else obj.index
         other_axis = 1 if self.axis == 0 else 0  # switches between 0 & 1
         concatenated = concat(applied, axis=self.axis, verify_integrity=False)
@@ -1853,3 +1846,28 @@ def func(df):
         return self._python_apply_general(func, self._obj_with_exclusions)
 
     boxplot = boxplot_frame_groupby
+
+
+def _wrap_transform_general_frame(
+    obj: DataFrame, group: DataFrame, res: DataFrame | Series
+) -> DataFrame:
+    from pandas import concat
+
+    if isinstance(res, Series):
+        # we need to broadcast across the
+        # other dimension; this will preserve dtypes
+        # GH14457
+        if res.index.is_(obj.index):
+            res_frame = concat([res] * len(group.columns), axis=1)
+            res_frame.columns = group.columns
+            res_frame.index = group.index
+        else:
+            res_frame = obj._constructor(
+                np.concatenate([res.values] * len(group.index)).reshape(group.shape),
+                columns=group.columns,
+                index=group.index,
+            )
+        assert isinstance(res_frame, DataFrame)
+        return res_frame
+    else:
+        return res