Merge remote-tracking branch 'upstream/main' into roll_var_remove_floating_point_artifacts

# Conflicts:
#	doc/source/whatsnew/v1.5.0.rst

Author: auderson

.github/workflows/code-checks.yml

@@ -140,22 +140,8 @@ jobs:
     - name: Run ASV benchmarks
       run: |
         cd asv_bench
-        asv check -E existing
-        git remote add upstream https://github.com/pandas-dev/pandas.git
-        git fetch upstream
         asv machine --yes
-        asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
-        if grep "failed" benchmarks.log > /dev/null ; then
-            exit 1
-        fi
-      if: ${{ steps.build.outcome == 'success' }}
-
-    - name: Publish benchmarks artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: Benchmarks log
-        path: asv_bench/benchmarks.log
-      if: failure()
+        asv run --quick --dry-run --strict --durations=30 --python=same
 
   build_docker_dev_environment:
     name: Build Docker Dev Environment

.pre-commit-config.yaml

@@ -70,6 +70,10 @@ repos:
       - id: rst-inline-touching-normal
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
+-   repo: https://github.com/sphinx-contrib/sphinx-lint
+    rev: v0.2
+    hooks:
+    - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
     rev: v1.3.0
     hooks:

LICENSES/OTHER

@@ -1,8 +1,3 @@
-numpydoc license
-----------------
-
-The numpydoc license is in pandas/doc/sphinxext/LICENSE.txt
-
 Bottleneck license
 ------------------
 
@@ -77,4 +72,4 @@ DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ci/deps/actions-310.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/actions-38-downstream_compat.yaml

@@ -10,7 +10,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -28,6 +27,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml
@@ -60,7 +60,7 @@ dependencies:
   - cftime
   - dask
   - ipython
-  - geopandas
+  - geopandas-base
   - seaborn
   - scikit-learn
   - statsmodels

ci/deps/actions-38-minimum_versions.yaml

@@ -11,7 +11,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -29,6 +28,7 @@ dependencies:
   - fastparquet=0.4.0
   - fsspec=0.7.4
   - html5lib=1.1
+  - hypothesis=5.5.3
   - gcsfs=0.6.0
   - jinja2=2.11
   - lxml=4.5.0

ci/deps/actions-38.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/actions-39.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

ci/deps/circle-38-arm64.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - hypothesis>=5.5.3
   - psutil
   - pytest-asyncio>=0.17
   - boto3
@@ -27,6 +26,7 @@ dependencies:
   - fastparquet
   - fsspec
   - html5lib
+  - hypothesis
   - gcsfs
   - jinja2
   - lxml

doc/source/development/contributing_codebase.rst

@@ -223,7 +223,7 @@ In some cases you may be tempted to use ``cast`` from the typing module when you
            ...
        else:  # Reasonably only str objects would reach this but...
            obj = cast(str, obj)  # Mypy complains without this!
-	   return obj.upper()
+           return obj.upper()
 
 The limitation here is that while a human can reasonably understand that ``is_number`` would catch the ``int`` and ``float`` types mypy cannot make that same inference just yet (see `mypy #5206 <https://github.com/python/mypy/issues/5206>`_. While the above works, the use of ``cast`` is **strongly discouraged**. Where applicable a refactor of the code to appease static analysis is preferable
 

doc/source/development/contributing_environment.rst

@@ -85,10 +85,10 @@ You will need `Build Tools for Visual Studio 2019
 <https://visualstudio.microsoft.com/downloads/>`_.
 
 .. warning::
-	You DO NOT need to install Visual Studio 2019.
-	You only need "Build Tools for Visual Studio 2019" found by
-	scrolling down to "All downloads" -> "Tools for Visual Studio 2019".
-	In the installer, select the "C++ build tools" workload.
+        You DO NOT need to install Visual Studio 2019.
+        You only need "Build Tools for Visual Studio 2019" found by
+        scrolling down to "All downloads" -> "Tools for Visual Studio 2019".
+        In the installer, select the "C++ build tools" workload.
 
 You can install the necessary components on the commandline using
 `vs_buildtools.exe <https://download.visualstudio.microsoft.com/download/pr/9a26f37e-6001-429b-a5db-c5455b93953c/460d80ab276046de2455a4115cc4e2f1e6529c9e6cb99501844ecafd16c619c4/vs_BuildTools.exe>`_:

doc/source/ecosystem.rst

@@ -540,15 +540,15 @@ Pandas-Genomics provides extension types, extension arrays, and extension access
 `Pint-Pandas`_
 ~~~~~~~~~~~~~~
 
-``Pint-Pandas <https://github.com/hgrecco/pint-pandas>`` provides an extension type for
+`Pint-Pandas <https://github.com/hgrecco/pint-pandas>`_ provides an extension type for
 storing numeric arrays with units. These arrays can be stored inside pandas'
 Series and DataFrame. Operations between Series and DataFrame columns which
 use pint's extension array are then units aware.
 
 `Text Extensions for Pandas`_
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``Text Extensions for Pandas <https://ibm.biz/text-extensions-for-pandas>``
+`Text Extensions for Pandas <https://ibm.biz/text-extensions-for-pandas>`_
 provides extension types to cover common data structures for representing natural language
 data, plus library integrations that convert the outputs of popular natural language
 processing libraries into Pandas DataFrames.

doc/source/reference/groupby.rst

@@ -132,9 +132,7 @@ The following methods are available only for ``SeriesGroupBy`` objects.
    SeriesGroupBy.hist
    SeriesGroupBy.nlargest
    SeriesGroupBy.nsmallest
-   SeriesGroupBy.nunique
    SeriesGroupBy.unique
-   SeriesGroupBy.value_counts
    SeriesGroupBy.is_monotonic_increasing
    SeriesGroupBy.is_monotonic_decreasing
 

doc/source/reference/series.rst

@@ -342,6 +342,7 @@ Datetime methods
    :toctree: api/
    :template: autosummary/accessor_method.rst
 
+   Series.dt.isocalendar
    Series.dt.to_period
    Series.dt.to_pydatetime
    Series.dt.tz_localize

doc/source/user_guide/advanced.rst

@@ -1082,14 +1082,14 @@ of :ref:`frequency aliases <timeseries.offset_aliases>` with datetime-like inter
 
    pd.interval_range(start=pd.Timedelta("0 days"), periods=3, freq="9H")
 
-Additionally, the ``closed`` parameter can be used to specify which side(s) the intervals
-are closed on.  Intervals are closed on the right side by default.
+Additionally, the ``inclusive`` parameter can be used to specify which side(s) the intervals
+are closed on.  Intervals are closed on the both side by default.
 
 .. ipython:: python
 
-   pd.interval_range(start=0, end=4, closed="both")
+   pd.interval_range(start=0, end=4, inclusive="both")
 
-   pd.interval_range(start=0, end=4, closed="neither")
+   pd.interval_range(start=0, end=4, inclusive="neither")
 
 Specifying ``start``, ``end``, and ``periods`` will generate a range of evenly spaced
 intervals from ``start`` to ``end`` inclusively, with ``periods`` number of elements

doc/source/user_guide/cookbook.rst

@@ -423,7 +423,7 @@ Fill forward a reversed timeseries
    )
    df.loc[df.index[3], "A"] = np.nan
    df
-   df.reindex(df.index[::-1]).ffill()
+   df.bfill()
 
 `cumsum reset at NaN values
 <https://stackoverflow.com/questions/18196811/cumsum-reset-at-nan>`__

doc/source/user_guide/dsintro.rst

@@ -678,7 +678,7 @@ Boolean operators operate element-wise as well:
 Transposing
 ~~~~~~~~~~~
 
-To transpose, access the ``T`` attribute or :meth:`DataFrame.transpose``,
+To transpose, access the ``T`` attribute or :meth:`DataFrame.transpose`,
 similar to an ndarray:
 
 .. ipython:: python

doc/source/user_guide/groupby.rst

@@ -539,19 +539,19 @@ Some common aggregating functions are tabulated below:
     :widths: 20, 80
     :delim: ;
 
-	:meth:`~pd.core.groupby.DataFrameGroupBy.mean`;Compute mean of groups
-	:meth:`~pd.core.groupby.DataFrameGroupBy.sum`;Compute sum of group values
-	:meth:`~pd.core.groupby.DataFrameGroupBy.size`;Compute group sizes
-	:meth:`~pd.core.groupby.DataFrameGroupBy.count`;Compute count of group
-	:meth:`~pd.core.groupby.DataFrameGroupBy.std`;Standard deviation of groups
-	:meth:`~pd.core.groupby.DataFrameGroupBy.var`;Compute variance of groups
-	:meth:`~pd.core.groupby.DataFrameGroupBy.sem`;Standard error of the mean of groups
-	:meth:`~pd.core.groupby.DataFrameGroupBy.describe`;Generates descriptive statistics
-	:meth:`~pd.core.groupby.DataFrameGroupBy.first`;Compute first of group values
-	:meth:`~pd.core.groupby.DataFrameGroupBy.last`;Compute last of group values
-	:meth:`~pd.core.groupby.DataFrameGroupBy.nth`;Take nth value, or a subset if n is a list
-	:meth:`~pd.core.groupby.DataFrameGroupBy.min`;Compute min of group values
-	:meth:`~pd.core.groupby.DataFrameGroupBy.max`;Compute max of group values
+        :meth:`~pd.core.groupby.DataFrameGroupBy.mean`;Compute mean of groups
+        :meth:`~pd.core.groupby.DataFrameGroupBy.sum`;Compute sum of group values
+        :meth:`~pd.core.groupby.DataFrameGroupBy.size`;Compute group sizes
+        :meth:`~pd.core.groupby.DataFrameGroupBy.count`;Compute count of group
+        :meth:`~pd.core.groupby.DataFrameGroupBy.std`;Standard deviation of groups
+        :meth:`~pd.core.groupby.DataFrameGroupBy.var`;Compute variance of groups
+        :meth:`~pd.core.groupby.DataFrameGroupBy.sem`;Standard error of the mean of groups
+        :meth:`~pd.core.groupby.DataFrameGroupBy.describe`;Generates descriptive statistics
+        :meth:`~pd.core.groupby.DataFrameGroupBy.first`;Compute first of group values
+        :meth:`~pd.core.groupby.DataFrameGroupBy.last`;Compute last of group values
+        :meth:`~pd.core.groupby.DataFrameGroupBy.nth`;Take nth value, or a subset if n is a list
+        :meth:`~pd.core.groupby.DataFrameGroupBy.min`;Compute min of group values
+        :meth:`~pd.core.groupby.DataFrameGroupBy.max`;Compute max of group values
 
 
 The aggregating functions above will exclude NA values. Any function which
@@ -1052,7 +1052,14 @@ Some operations on the grouped data might not fit into either the aggregate or
 transform categories. Or, you may simply want GroupBy to infer how to combine
 the results. For these, use the ``apply`` function, which can be substituted
 for both ``aggregate`` and ``transform`` in many standard use cases. However,
-``apply`` can handle some exceptional use cases, for example:
+``apply`` can handle some exceptional use cases.
+
+.. note::
+
+   ``apply`` can act as a reducer, transformer, *or* filter function, depending
+   on exactly what is passed to it. It can depend on the passed function and
+   exactly what you are grouping. Thus the grouped column(s) may be included in
+   the output as well as set the indices.
 
 .. ipython:: python
 
@@ -1064,16 +1071,14 @@ for both ``aggregate`` and ``transform`` in many standard use cases. However,
 
 The dimension of the returned result can also change:
 
-.. ipython::
-
-    In [8]: grouped = df.groupby('A')['C']
+.. ipython:: python
 
-    In [10]: def f(group):
-       ....:     return pd.DataFrame({'original': group,
-       ....:                          'demeaned': group - group.mean()})
-       ....:
+    grouped = df.groupby('A')['C']
 
-    In [11]: grouped.apply(f)
+    def f(group):
+        return pd.DataFrame({'original': group,
+                             'demeaned': group - group.mean()})
+    grouped.apply(f)
 
 ``apply`` on a Series can operate on a returned value from the applied function,
 that is itself a series, and possibly upcast the result to a DataFrame:
@@ -1088,11 +1093,33 @@ that is itself a series, and possibly upcast the result to a DataFrame:
     s
     s.apply(f)
 
+Control grouped column(s) placement with ``group_keys``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 .. note::
 
-   ``apply`` can act as a reducer, transformer, *or* filter function, depending on exactly what is passed to it.
-   So depending on the path taken, and exactly what you are grouping. Thus the grouped columns(s) may be included in
-   the output as well as set the indices.
+   If ``group_keys=True`` is specified when calling :meth:`~DataFrame.groupby`,
+   functions passed to ``apply`` that return like-indexed outputs will have the
+   group keys added to the result index. Previous versions of pandas would add
+   the group keys only when the result from the applied function had a different
+   index than the input. If ``group_keys`` is not specified, the group keys will
+   not be added for like-indexed outputs. In the future this behavior
+   will change to always respect ``group_keys``, which defaults to ``True``.
+
+   .. versionchanged:: 1.5.0
+
+To control whether the grouped column(s) are included in the indices, you can use
+the argument ``group_keys``. Compare
+
+.. ipython:: python
+
+    df.groupby("A", group_keys=True).apply(lambda x: x)
+
+with
+
+.. ipython:: python
+
+    df.groupby("A", group_keys=False).apply(lambda x: x)
 
 Similar to :ref:`groupby.aggregate.udfs`, the resulting dtype will reflect that of the
 apply function. If the results from different groups have different dtypes, then

doc/source/user_guide/io.rst

@@ -5695,9 +5695,9 @@ for an explanation of how the database connection is handled.
 
 .. warning::
 
-	When you open a connection to a database you are also responsible for closing it.
-	Side effects of leaving a connection open may include locking the database or
-	other breaking behaviour.
+        When you open a connection to a database you are also responsible for closing it.
+        Side effects of leaving a connection open may include locking the database or
+        other breaking behaviour.
 
 Writing DataFrames
 ''''''''''''''''''

doc/source/user_guide/timeseries.rst

@@ -2405,9 +2405,9 @@ you can use the ``tz_convert`` method.
 
 .. warning::
 
-	Be wary of conversions between libraries. For some time zones, ``pytz`` and ``dateutil`` have different
-	definitions of the zone. This is more of a problem for unusual time zones than for
-	'standard' zones like ``US/Eastern``.
+        Be wary of conversions between libraries. For some time zones, ``pytz`` and ``dateutil`` have different
+        definitions of the zone. This is more of a problem for unusual time zones than for
+        'standard' zones like ``US/Eastern``.
 
 .. warning::