Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

.github/workflows/code-checks.yml

@@ -80,6 +80,10 @@ jobs:
       id: build
       uses: ./.github/actions/build_pandas
 
+    - name: Check for no warnings when building single-page docs
+      run: ci/code_checks.sh single-docs
+      if: ${{ steps.build.outcome == 'success' }}
+
     - name: Run checks on imported code
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' }}

.github/workflows/docbuild-and-upload.yml

@@ -37,6 +37,7 @@ jobs:
       run: |
         source activate pandas-dev
         python web/pandas_web.py web/pandas --target-path=web/build
+
     - name: Build documentation
       run: |
         source activate pandas-dev

.github/workflows/posix.yml

@@ -28,7 +28,7 @@ jobs:
         pattern: ["not single_cpu", "single_cpu"]
         # Don't test pyarrow v2/3: Causes timeouts in read_csv engine
         # even if tests are skipped/xfailed
-        pyarrow_version: ["5", "7"]
+        pyarrow_version: ["5", "6", "7"]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -62,6 +62,15 @@ jobs:
             pattern: "not slow and not network and not single_cpu"
             pandas_testing_mode: "deprecate"
             test_args: "-W error::DeprecationWarning:numpy"
+        exclude:
+          - env_file: actions-39.yaml
+            pyarrow_version: "6"
+          - env_file: actions-39.yaml
+            pyarrow_version: "7"
+          - env_file: actions-310.yaml
+            pyarrow_version: "6"
+          - env_file: actions-310.yaml
+            pyarrow_version: "7"
       fail-fast: false
     name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
     env:

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         types_or: [python, rst, markdown]
         files: ^(pandas|doc)/
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.2.0
     hooks:
     -   id: debug-statements
     -   id: end-of-file-fixer
@@ -56,7 +56,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.1
+    rev: v2.32.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -71,7 +71,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.2
+    rev: v0.4.1
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -11,10 +11,14 @@
 
 try:
     old_sig = False
-    from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    from pandas._libs.tslibs import tz_convert_from_utc
 except ImportError:
-    old_sig = True
-    from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
+    try:
+        old_sig = False
+        from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    except ImportError:
+        old_sig = True
+        from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
 
 
 class TimeTZConvert:

azure-pipelines.yml

@@ -48,6 +48,7 @@ jobs:
         pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
+        export PANDAS_CI=1 && \
         pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'
 

ci/code_checks.sh

@@ -12,8 +12,9 @@
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh typing        # run static type analysis
+#   $ ./ci/code_checks.sh single-docs   # check single-page docs build warning-free
 
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" ]] || \
+[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" || "$1" == "single-docs" ]] || \
     { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
@@ -102,4 +103,11 @@ if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
     fi
 fi
 
+### SINGLE-PAGE DOCS ###
+if [[ -z "$CHECK" || "$CHECK" == "single-docs" ]]; then
+    python doc/make.py --warnings-are-errors --single pandas.Series.value_counts
+    python doc/make.py --warnings-are-errors --single pandas.Series.str.split
+    python doc/make.py clean
+fi
+
 exit $RET

doc/source/development/policies.rst

@@ -51,7 +51,7 @@ pandas may change the behavior of experimental features at any time.
 Python support
 ~~~~~~~~~~~~~~
 
-pandas will only drop support for specific Python versions (e.g. 3.6.x, 3.7.x) in
-pandas **major** or **minor** releases.
+pandas mirrors the `NumPy guidelines for Python support <https://numpy.org/neps/nep-0029-deprecation_policy.html#implementation>`__.
+
 
 .. _SemVer: https://semver.org

doc/source/getting_started/intro_tutorials/04_plotting.rst

@@ -2,6 +2,12 @@
 
 {{ header }}
 
+How do I create plots in pandas?
+----------------------------------
+
+.. image:: ../../_static/schemas/04_plot_overview.svg
+   :align: center
+
 .. ipython:: python
 
     import pandas as pd
@@ -35,12 +41,6 @@
     </ul>
     </div>
 
-How to create plots in pandas?
-------------------------------
-
-.. image:: ../../_static/schemas/04_plot_overview.svg
-   :align: center
-
 .. raw:: html
 
     <ul class="task-bullet">
@@ -52,6 +52,7 @@ I want a quick visual check of the data.
 
     @savefig 04_airqual_quick.png
     air_quality.plot()
+    plt.show()
 
 With a ``DataFrame``, pandas creates by default one line plot for each of
 the columns with numeric data.
@@ -68,10 +69,19 @@ the columns with numeric data.
 
 I want to plot only the columns of the data table with the data from Paris.
 
+.. ipython:: python
+   :suppress:
+
+    # We need to clear the figure here as, within doc generation, the plot
+    # accumulates data on each plot(). This is not needed when running
+    # in a notebook, so is suppressed from output.
+    plt.clf()
+
 .. ipython:: python
 
     @savefig 04_airqual_paris.png
     air_quality["station_paris"].plot()
+    plt.show()
 
 To plot a specific column, use the selection method of the
 :ref:`subset data tutorial <10min_tut_03_subset>` in combination with the :meth:`~DataFrame.plot`
@@ -94,6 +104,7 @@ I want to visually compare the :math:`NO_2` values measured in London versus Par
 
     @savefig 04_airqual_scatter.png
     air_quality.plot.scatter(x="station_london", y="station_paris", alpha=0.5)
+    plt.show()
 
 .. raw:: html
 
@@ -125,6 +136,7 @@ method is applicable on the air quality example data:
 
     @savefig 04_airqual_boxplot.png
     air_quality.plot.box()
+    plt.show()
 
 .. raw:: html
 
@@ -148,6 +160,7 @@ I want each of the columns in a separate subplot.
 
     @savefig 04_airqual_area_subplot.png
     axs = air_quality.plot.area(figsize=(12, 4), subplots=True)
+    plt.show()
 
 Separate subplots for each of the data columns are supported by the ``subplots`` argument
 of the ``plot`` functions. The builtin options available in each of the pandas plot
@@ -180,9 +193,10 @@ I want to further customize, extend or save the resulting plot.
 
     fig, axs = plt.subplots(figsize=(12, 4))
     air_quality.plot.area(ax=axs)
-    @savefig 04_airqual_customized.png
     axs.set_ylabel("NO$_2$ concentration")
+    @savefig 04_airqual_customized.png
     fig.savefig("no2_concentrations.png")
+    plt.show()
 
 .. ipython:: python
    :suppress:
@@ -208,6 +222,7 @@ This strategy is applied in the previous example:
    air_quality.plot.area(ax=axs)                   # Use pandas to put the area plot on the prepared Figure/Axes
    axs.set_ylabel("NO$_2$ concentration")          # Do any Matplotlib customization you like
    fig.savefig("no2_concentrations.png")           # Save the Figure/Axes using the existing Matplotlib method.
+   plt.show()                                      # Display the plot
 
 .. raw:: html
 

doc/source/index.rst.template

@@ -26,6 +26,7 @@ pandas documentation
 easy-to-use data structures and data analysis tools for the `Python <https://www.python.org/>`__
 programming language.
 
+{% if not single_doc -%}
 .. panels::
     :card: + intro-card text-center
     :column: col-lg-6 col-md-6 col-sm-6 col-xs-12 d-flex
@@ -96,16 +97,22 @@ programming language.
             :text: To the development guide
             :classes: btn-block btn-secondary stretched-link
 
-
+{% endif %}
 {% if single_doc and single_doc.endswith('.rst') -%}
 .. toctree::
     :maxdepth: 3
     :titlesonly:
 
     {{ single_doc[:-4] }}
+{% elif single_doc and single_doc.count('.') <= 1 %}
+.. autosummary::
+    :toctree: reference/api/
+
+    {{ single_doc }}
 {% elif single_doc %}
 .. autosummary::
     :toctree: reference/api/
+    :template: autosummary/accessor_method.rst
 
     {{ single_doc }}
 {% else -%}

doc/source/reference/frame.rst

@@ -391,3 +391,4 @@ Serialization / IO / conversion
    DataFrame.to_clipboard
    DataFrame.to_markdown
    DataFrame.style
+   DataFrame.__dataframe__

doc/source/reference/general_functions.rst

@@ -78,3 +78,10 @@ Hashing
 
    util.hash_array
    util.hash_pandas_object
+
+Importing from other DataFrame libraries
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+   api.exchange.from_dataframe

doc/source/user_guide/gotchas.rst

@@ -367,7 +367,7 @@ integer arrays to floating when NAs must be introduced.
 Differences with NumPy
 ----------------------
 For :class:`Series` and :class:`DataFrame` objects, :meth:`~DataFrame.var` normalizes by
-``N-1`` to produce unbiased estimates of the sample variance, while NumPy's
+``N-1`` to produce `unbiased estimates of the population variance <https://en.wikipedia.org/wiki/Bias_of_an_estimator>`__, while NumPy's
 :meth:`numpy.var` normalizes by N, which measures the variance of the sample. Note that
 :meth:`~DataFrame.cov` normalizes by ``N-1`` in both pandas and NumPy.