Merge branch 'master' of https://github.com/pandas-dev/pandas into BUG29751

Author: kasim95

.pre-commit-config.yaml

@@ -20,11 +20,9 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.6.4
+    rev: 5.7.0
     hooks:
     -   id: isort
-        types: [text] # overwrite upstream `types: [python]`
-        types_or: [python, cython]
 -   repo: https://github.com/asottile/pyupgrade
     rev: v2.7.4
     hooks:
@@ -128,7 +126,7 @@ repos:
         entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
         types: [python]
         files: ^pandas/tests/
-        exclude: ^pandas/tests/(computation|extension|io)/
+        exclude: ^pandas/tests/extension/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py

asv_bench/benchmarks/rolling.py

@@ -50,20 +50,24 @@ class Engine:
         ["int", "float"],
         [np.sum, lambda x: np.sum(x) + 5],
         ["cython", "numba"],
+        ["sum", "max", "min", "median", "mean"],
     )
-    param_names = ["constructor", "dtype", "function", "engine"]
+    param_names = ["constructor", "dtype", "function", "engine", "method"]
 
-    def setup(self, constructor, dtype, function, engine):
+    def setup(self, constructor, dtype, function, engine, method):
         N = 10 ** 3
         arr = (100 * np.random.random(N)).astype(dtype)
         self.data = getattr(pd, constructor)(arr)
 
-    def time_rolling_apply(self, constructor, dtype, function, engine):
+    def time_rolling_apply(self, constructor, dtype, function, engine, method):
         self.data.rolling(10).apply(function, raw=True, engine=engine)
 
-    def time_expanding_apply(self, constructor, dtype, function, engine):
+    def time_expanding_apply(self, constructor, dtype, function, engine, method):
         self.data.expanding().apply(function, raw=True, engine=engine)
 
+    def time_rolling_methods(self, constructor, dtype, function, engine, method):
+        getattr(self.data.rolling(10), method)(engine=engine)
+
 
 class ExpandingMethods:
 

ci/deps/actions-37-locale.yaml

@@ -30,7 +30,7 @@ dependencies:
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
-  - pyarrow>=0.17
+  - pyarrow=0.17 # GH 38803
   - pytables>=3.5.1
   - scipy
   - xarray=0.12.3

doc/source/_static/spreadsheets/conditional.png

@@ -432,7 +432,7 @@
 
 
 ipython_warning_is_error = False
-ipython_exec_lines = [
+ipython_execlines = [
     "import numpy as np",
     "import pandas as pd",
     # This ensures correct rendering on system with console encoding != utf8

doc/source/_static/spreadsheets/filter.png

@@ -698,6 +698,12 @@ to run its checks with::
 
 without needing to have done ``pre-commit install`` beforehand.
 
+If you want to run checks on all recently commited files on upstream/master you can use::
+
+    pre-commit run --from-ref=upstream/master --to-ref=HEAD --all-files
+
+without needing to have done ``pre-commit install`` beforehand.
+
 .. note::
 
     If you have conflicting installations of ``virtualenv``, then you may get an

doc/source/_static/spreadsheets/find.png

@@ -4,23 +4,13 @@
 
 Comparison with SAS
 ********************
+
 For potential users coming from `SAS <https://en.wikipedia.org/wiki/SAS_(software)>`__
 this page is meant to demonstrate how different SAS operations would be
 performed in pandas.
 
 .. include:: includes/introduction.rst
 
-.. note::
-
-   Throughout this tutorial, the pandas ``DataFrame`` will be displayed by calling
-   ``df.head()``, which displays the first N (default 5) rows of the ``DataFrame``.
-   This is often used in interactive work (e.g. `Jupyter notebook
-   <https://jupyter.org/>`_ or terminal) - the equivalent in SAS would be:
-
-   .. code-block:: sas
-
-      proc print data=df(obs=5);
-      run;
 
 Data structures
 ---------------
@@ -120,7 +110,7 @@ The pandas method is :func:`read_csv`, which works similarly.
        "pandas/master/pandas/tests/io/data/csv/tips.csv"
    )
    tips = pd.read_csv(url)
-   tips.head()
+   tips
 
 
 Like ``PROC IMPORT``, ``read_csv`` can take a number of parameters to specify
@@ -138,6 +128,19 @@ In addition to text/csv, pandas supports a variety of other data formats
 such as Excel, HDF5, and SQL databases.  These are all read via a ``pd.read_*``
 function.  See the :ref:`IO documentation<io>` for more details.
 
+Limiting output
+~~~~~~~~~~~~~~~
+
+.. include:: includes/limit.rst
+
+The equivalent in SAS would be:
+
+.. code-block:: sas
+
+   proc print data=df(obs=5);
+   run;
+
+
 Exporting data
 ~~~~~~~~~~~~~~
 
@@ -173,20 +176,8 @@ be used on new or existing columns.
        new_bill = total_bill / 2;
    run;
 
-pandas provides similar vectorized operations by
-specifying the individual ``Series`` in the ``DataFrame``.
-New columns can be assigned in the same way.
+.. include:: includes/column_operations.rst
 
-.. ipython:: python
-
-   tips["total_bill"] = tips["total_bill"] - 2
-   tips["new_bill"] = tips["total_bill"] / 2.0
-   tips.head()
-
-.. ipython:: python
-   :suppress:
-
-   tips = tips.drop("new_bill", axis=1)
 
 Filtering
 ~~~~~~~~~
@@ -278,18 +269,7 @@ drop, and rename columns.
        rename total_bill=total_bill_2;
    run;
 
-The same operations are expressed in pandas below.
-
-.. ipython:: python
-
-   # keep
-   tips[["sex", "total_bill", "tip"]].head()
-
-   # drop
-   tips.drop("sex", axis=1).head()
-
-   # rename
-   tips.rename(columns={"total_bill": "total_bill_2"}).head()
+.. include:: includes/column_selection.rst
 
 
 Sorting by values
@@ -442,6 +422,8 @@ input frames.
 Missing data
 ------------
 
+Both pandas and SAS have a representation for missing data.
+
 .. include:: includes/missing_intro.rst
 
 One difference is that missing data cannot be compared to its sentinel value.