Merge remote-tracking branch 'upstream/master' into astype-keep-name

Author: dsaxton

.github/workflows/ci.yml

@@ -125,68 +125,32 @@ jobs:
     - name: Check ipython directive errors
       run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
 
-    - name: Merge website and docs
-      run: |
-        mkdir -p pandas_web/docs
-        cp -r web/build/* pandas_web/
-        cp -r doc/build/html/* pandas_web/docs/
-      if: github.event_name == 'push'
-
     - name: Install Rclone
       run: sudo apt install rclone -y
       if: github.event_name == 'push'
 
     - name: Set up Rclone
       run: |
-        RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf
-        mkdir -p `dirname $RCLONE_CONFIG_PATH`
-        echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH
-        echo "type = swift" >> $RCLONE_CONFIG_PATH
-        echo "env_auth = false" >> $RCLONE_CONFIG_PATH
-        echo "auth_version = 3" >> $RCLONE_CONFIG_PATH
-        echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH
-        echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH
-        echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH
-        echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH
-        echo "domain = default" >> $RCLONE_CONFIG_PATH
-        echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH
-        echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH
-        echo "region = BHS" >> $RCLONE_CONFIG_PATH
+        CONF=$HOME/.config/rclone/rclone.conf
+        mkdir -p `dirname $CONF`
+        echo "[ovh_host]" > $CONF
+        echo "type = swift" >> $CONF
+        echo "env_auth = false" >> $CONF
+        echo "auth_version = 3" >> $CONF
+        echo "auth = https://auth.cloud.ovh.net/v3/" >> $CONF
+        echo "endpoint_type = public" >> $CONF
+        echo "tenant_domain = default" >> $CONF
+        echo "tenant = 2977553886518025" >> $CONF
+        echo "domain = default" >> $CONF
+        echo "user = w4KGs3pmDxpd" >> $CONF
+        echo "key = ${{ secrets.ovh_object_store_key }}" >> $CONF
+        echo "region = BHS" >> $CONF
       if: github.event_name == 'push'
 
     - name: Sync web with OVH
-      run: rclone sync pandas_web ovh_cloud_pandas_web:dev
-      if: github.event_name == 'push'
-
-    - name: Create git repo to upload the built docs to GitHub pages
-      run: |
-        cd pandas_web
-        git init
-        touch .nojekyll
-        echo "dev.pandas.io" > CNAME
-        printf "User-agent: *\nDisallow: /" > robots.txt
-        git add --all .
-        git config user.email "[email protected]"
-        git config user.name "pandas-bot"
-        git commit -m "pandas web and documentation in master"
+      run: rclone sync --exclude pandas-docs/** web/build ovh_host:prod
       if: github.event_name == 'push'
 
-    # For this task to work, next steps are required:
-    # 1. Generate a pair of private/public keys (i.e. `ssh-keygen -t rsa -b 4096 -C "[email protected]"`)
-    # 2. Go to https://github.com/pandas-dev/pandas/settings/secrets
-    # 3. Click on "Add a new secret"
-    # 4. Name: "github_pagas_ssh_key", Value: <Content of the private ssh key>
-    # 5. The public key needs to be upladed to https://github.com/pandas-dev/pandas-dev.github.io/settings/keys
-    - name: Install GitHub pages ssh deployment key
-      uses: shimataro/ssh-key-action@v2
-      with:
-        key: ${{ secrets.github_pages_ssh_key }}
-        known_hosts: 'github.com,192.30.252.128 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=='
-      if: github.event_name == 'push'
-
-    - name: Publish web and docs to GitHub pages
-      run: |
-        cd pandas_web
-        git remote add origin [email protected]:pandas-dev/pandas-dev.github.io.git
-        git push -f origin master || true
+    - name: Sync dev docs with OVH
+      run: rclone sync doc/build/html ovh_host:prod/pandas-docs/dev
       if: github.event_name == 'push'

doc/source/development/meeting.rst

@@ -25,7 +25,7 @@ This calendar shows all the developer meetings.
 You can subscribe to this calendar with the following links:
 
 * `iCal <https://calendar.google.com/calendar/ical/pgbn14p6poja8a1cf2dv2jhrmg%40group.calendar.google.com/public/basic.ics>`__
-* `Google calendar <https://calendar.google.com/calendar/embed?src=pgbn14p6poja8a1cf2dv2jhrmg%40group.calendar.google.com>`__
+* `Google calendar <https://calendar.google.com/calendar/r?cid=pgbn14p6poja8a1cf2dv2jhrmg@group.calendar.google.com>`__
 
 Additionally, we'll sometimes have one-off meetings on specific topics.
 These will be published on the same calendar.

doc/source/whatsnew/v1.0.2.rst

@@ -17,16 +17,17 @@ Fixed regressions
 
 - Fixed regression in :meth:`DataFrame.to_excel` when ``columns`` kwarg is passed (:issue:`31677`)
 - Fixed regression in :meth:`Series.align` when ``other`` is a DataFrame and ``method`` is not None (:issue:`31785`)
-- Fixed regression in :meth:`pandas.core.groupby.RollingGroupby.apply` where the ``raw`` parameter was ignored (:issue:`31754`)
-- Fixed regression in :meth:`pandas.core.window.Rolling.corr` when using a time offset (:issue:`31789`)
-- Fixed regression in :meth:`pandas.core.groupby.DataFrameGroupBy.nunique` which was modifying the original values if ``NaN`` values were present (:issue:`31950`)
+- Fixed regression in ``groupby(..).rolling(..).apply()`` (``RollingGroupby``) where the ``raw`` parameter was ignored (:issue:`31754`)
+- Fixed regression in :meth:`rolling(..).corr() <pandas.core.window.rolling.Rolling.corr>` when using a time offset (:issue:`31789`)
+- Fixed regression in :meth:`groupby(..).nunique() <pandas.core.groupby.DataFrameGroupBy.nunique>` which was modifying the original values if ``NaN`` values were present (:issue:`31950`)
 - Fixed regression where :func:`read_pickle` raised a ``UnicodeDecodeError`` when reading a py27 pickle with :class:`MultiIndex` column (:issue:`31988`).
 - Fixed regression in :class:`DataFrame` arithmetic operations with mis-matched columns (:issue:`31623`)
-- Fixed regression in :meth:`pandas.core.groupby.GroupBy.agg` calling a user-provided function an extra time on an empty input (:issue:`31760`)
-- Joining on :class:`DatetimeIndex` or :class:`TimedeltaIndex` will preserve ``freq`` in simple cases (:issue:`32166`)
-- Fixed bug in the repr of an object-dtype :class:`Index` with bools and missing values (:issue:`32146`)
+- Fixed regression in :meth:`groupby(..).agg() <pandas.core.groupby.GroupBy.agg>` calling a user-provided function an extra time on an empty input (:issue:`31760`)
+- Fixed regression in joining on :class:`DatetimeIndex` or :class:`TimedeltaIndex` to preserve ``freq`` in simple cases (:issue:`32166`)
+- Fixed regression in the repr of an object-dtype :class:`Index` with bools and missing values (:issue:`32146`)
 - Fixed regression in :meth:`read_csv` in which the ``encoding`` option was not recognized with certain file-like objects (:issue:`31819`)
--
+- Fixed regression in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with (tz-aware) index and ``method=nearest`` (:issue:`26683`)
+
 
 .. ---------------------------------------------------------------------------
 
@@ -64,7 +65,6 @@ Bug fixes
 
 **Datetimelike**
 
-- Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` when reindexing with a tz-aware index (:issue:`26683`)
 - Bug in :meth:`Series.astype` not copying for tz-naive and tz-aware datetime64 dtype (:issue:`32490`)
 - Bug where :func:`to_datetime` would raise when passed ``pd.NA`` (:issue:`32213`)
 - Improved error message when subtracting two :class:`Timestamp` that result in an out-of-bounds :class:`Timedelta` (:issue:`31774`)
@@ -78,16 +78,17 @@ Bug fixes
 **I/O**
 
 - Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`)
+- Bug in :meth:`pandas.json_normalize` when value in meta path is not iterable (:issue:`31507`)
 - Fixed pickling of ``pandas.NA``. Previously a new object was returned, which broke computations relying on ``NA`` being a singleton (:issue:`31847`)
 - Fixed bug in parquet roundtrip with nullable unsigned integer dtypes (:issue:`31896`).
 
 **Experimental dtypes**
 
-- Fix bug in :meth:`DataFrame.convert_dtypes` for columns that were already using the ``"string"`` dtype (:issue:`31731`).
+- Fixed bug in :meth:`DataFrame.convert_dtypes` for columns that were already using the ``"string"`` dtype (:issue:`31731`).
+- Fixed bug in :meth:`DataFrame.convert_dtypes` for series with mix of integers and strings (:issue:`32117`)
+- Fixed bug in :meth:`DataFrame.convert_dtypes` where ``BooleanDtype`` columns were converted to ``Int64`` (:issue:`32287`)
 - Fixed bug in setting values using a slice indexer with string dtype (:issue:`31772`)
 - Fixed bug where :meth:`pandas.core.groupby.GroupBy.first` and :meth:`pandas.core.groupby.GroupBy.last` would raise a ``TypeError`` when groups contained ``pd.NA`` in a column of object dtype (:issue:`32123`)
-- Fix bug in :meth:`Series.convert_dtypes` for series with mix of integers and strings (:issue:`32117`)
-- Fixed bug in :meth:`DataFrame.convert_dtypes`, where ``BooleanDtype`` columns were converted to ``Int64`` (:issue:`32287`)
 
 **Strings**
 

doc/source/whatsnew/v1.1.0.rst

@@ -188,7 +188,9 @@ Performance improvements
 - Performance improvement in :class:`Timedelta` constructor (:issue:`30543`)
 - Performance improvement in :class:`Timestamp` constructor (:issue:`30543`)
 - Performance improvement in flex arithmetic ops between :class:`DataFrame` and :class:`Series` with ``axis=0`` (:issue:`31296`)
--
+- The internal :meth:`Index._shallow_copy` now copies cached attributes over to the new index,
+  avoiding creating these again on the new index. This can speed up many operations
+  that depend on creating copies of existing indexes (:issue:`28584`)
 
 .. ---------------------------------------------------------------------------
 
@@ -263,7 +265,7 @@ Indexing
 - Bug in :meth:`Series.xs` incorrectly returning ``Timestamp`` instead of ``datetime64`` in some object-dtype cases (:issue:`31630`)
 - Bug in :meth:`DataFrame.iat` incorrectly returning ``Timestamp`` instead of ``datetime`` in some object-dtype cases (:issue:`32809`)
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
--
+- Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
 
 Missing
 ^^^^^^^

pandas/core/arrays/categorical.py

@@ -1317,7 +1317,7 @@ def __setstate__(self, state):
             setattr(self, k, v)
 
     @property
-    def T(self):
+    def T(self) -> "Categorical":
         """
         Return transposed numpy array.
         """

pandas/core/arrays/sparse/array.py

@@ -1296,14 +1296,14 @@ def mean(self, axis=0, *args, **kwargs):
             nsparse = self.sp_index.ngaps
             return (sp_sum + self.fill_value * nsparse) / (ct + nsparse)
 
-    def transpose(self, *axes):
+    def transpose(self, *axes) -> "SparseArray":
         """
         Returns the SparseArray.
         """
         return self
 
     @property
-    def T(self):
+    def T(self) -> "SparseArray":
         """
         Returns the SparseArray.
         """

pandas/core/computation/pytables.py

@@ -17,6 +17,7 @@
 from pandas.core.computation.common import _ensure_decoded
 from pandas.core.computation.expr import BaseExprVisitor
 from pandas.core.computation.ops import UndefinedVariableError, is_term
+from pandas.core.construction import extract_array
 
 from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded
 
@@ -202,7 +203,7 @@ def stringify(value):
             v = Timedelta(v, unit="s").value
             return TermValue(int(v), v, kind)
         elif meta == "category":
-            metadata = com.values_from_object(self.metadata)
+            metadata = extract_array(self.metadata, extract_numpy=True)
             result = metadata.searchsorted(v, side="left")
 
             # result returns 0 if v is first element or if v is not in metadata

pandas/core/frame.py

@@ -77,6 +77,7 @@
     ensure_platform_int,
     infer_dtype_from_object,
     is_bool_dtype,
+    is_datetime64_any_dtype,
     is_dict_like,
     is_dtype_equal,
     is_extension_array_dtype,
@@ -88,6 +89,7 @@
     is_list_like,
     is_named_tuple,
     is_object_dtype,
+    is_period_dtype,
     is_scalar,
     is_sequence,
     needs_i8_conversion,
@@ -2443,7 +2445,9 @@ def transpose(self, *args, copy: bool = False) -> "DataFrame":
 
         return result.__finalize__(self)
 
-    T = property(transpose)
+    @property
+    def T(self) -> "DataFrame":
+        return self.transpose()
 
     # ----------------------------------------------------------------------
     # Indexing Methods
@@ -2706,6 +2710,20 @@ def _setitem_frame(self, key, value):
         self._check_setitem_copy()
         self._where(-key, value, inplace=True)
 
+    def _iset_item(self, loc: int, value):
+        self._ensure_valid_index(value)
+
+        # technically _sanitize_column expects a label, not a position,
+        #  but the behavior is the same as long as we pass broadcast=False
+        value = self._sanitize_column(loc, value, broadcast=False)
+        NDFrame._iset_item(self, loc, value)
+
+        # check if we are modifying a copy
+        # try to set first as we want an invalid
+        # value exception to occur first
+        if len(self):
+            self._check_setitem_copy()
+
     def _set_item(self, key, value):
         """
         Add series to DataFrame in specified column.
@@ -7789,11 +7807,13 @@ def _reduce(
         self, op, name, axis=0, skipna=True, numeric_only=None, filter_type=None, **kwds
     ):
 
-        dtype_is_dt = self.dtypes.apply(lambda x: x.kind == "M")
+        dtype_is_dt = self.dtypes.apply(
+            lambda x: is_datetime64_any_dtype(x) or is_period_dtype(x)
+        )
         if numeric_only is None and name in ["mean", "median"] and dtype_is_dt.any():
             warnings.warn(
                 "DataFrame.mean and DataFrame.median with numeric_only=None "
-                "will include datetime64 and datetime64tz columns in a "
+                "will include datetime64, datetime64tz, and PeriodDtype columns in a "
                 "future version.",
                 FutureWarning,
                 stacklevel=3,
@@ -7854,6 +7874,10 @@ def blk_func(values):
                 assert len(res) == max(list(res.keys())) + 1, res.keys()
             out = df._constructor_sliced(res, index=range(len(res)), dtype=out_dtype)
             out.index = df.columns
+            if axis == 0 and df.dtypes.apply(needs_i8_conversion).any():
+                # FIXME: needs_i8_conversion check is kludge, not sure
+                #  why it is necessary in this case and this case alone
+                out[:] = coerce_to_dtypes(out.values, df.dtypes)
             return out
 
         if numeric_only is None:

pandas/core/generic.py

@@ -967,7 +967,6 @@ def rename(
                 continue
 
             ax = self._get_axis(axis_no)
-            baxis = self._get_block_manager_axis(axis_no)
             f = com.get_rename_function(replacements)
 
             if level is not None:
@@ -984,9 +983,8 @@ def rename(
                     ]
                     raise KeyError(f"{missing_labels} not found in axis")
 
-            result._data = result._data.rename_axis(
-                f, axis=baxis, copy=copy, level=level
-            )
+            new_index = ax._transform_index(f, level)
+            result.set_axis(new_index, axis=axis_no, inplace=True)
             result._clear_item_cache()
 
         if inplace:
@@ -3579,6 +3577,10 @@ def _slice(self: FrameOrSeries, slobj: slice, axis=0) -> FrameOrSeries:
         result._set_is_copy(self, copy=is_copy)
         return result
 
+    def _iset_item(self, loc: int, value) -> None:
+        self._data.iset(loc, value)
+        self._clear_item_cache()
+
     def _set_item(self, key, value) -> None:
         self._data.set(key, value)
         self._clear_item_cache()

pandas/core/groupby/generic.py

@@ -49,7 +49,7 @@
     is_scalar,
     needs_i8_conversion,
 )
-from pandas.core.dtypes.missing import _isna_ndarraylike, isna, notna
+from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core.aggregation import (
     is_multi_agg_with_relabel,
@@ -1772,10 +1772,8 @@ def count(self):
         ids, _, ngroups = self.grouper.group_info
         mask = ids != -1
 
-        vals = (
-            (mask & ~_isna_ndarraylike(np.atleast_2d(blk.get_values())))
-            for blk in data.blocks
-        )
+        # TODO(2DEA): reshape would not be necessary with 2D EAs
+        vals = ((mask & ~isna(blk.values).reshape(blk.shape)) for blk in data.blocks)
         locs = (blk.mgr_locs for blk in data.blocks)
 
         counted = (

pandas/core/groupby/ops.py

@@ -217,7 +217,7 @@ def indices(self):
             return self.groupings[0].indices
         else:
             codes_list = [ping.codes for ping in self.groupings]
-            keys = [com.values_from_object(ping.group_index) for ping in self.groupings]
+            keys = [ping.group_index for ping in self.groupings]
             return get_indexer_dict(codes_list, keys)
 
     @property