Merge branch 'main' into Fix-pandas-dev#37715-column.py

Author: bang128

.github/workflows/code-checks.yml

@@ -39,8 +39,8 @@ jobs:
       with:
         extra_args: --verbose --all-files
 
-  docstring_typing_pylint:
-    name: Docstring validation, typing, and pylint
+  docstring_typing_manual_hooks:
+    name: Docstring validation, typing, and other manual pre-commit hooks
     runs-on: ubuntu-22.04
     defaults:
       run:

.pre-commit-config.yaml

@@ -1,7 +1,17 @@
 minimum_pre_commit_version: 2.15.0
 exclude: ^LICENSES/|\.(html|csv|svg)$
-# reserve "manual" for mypy and pyright
-default_stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
+# reserve "manual" for relatively slow hooks which we still want to run in CI
+default_stages: [
+    commit,
+    merge-commit,
+    push,
+    prepare-commit-msg,
+    commit-msg,
+    post-checkout,
+    post-commit,
+    post-merge,
+    post-rewrite
+]
 ci:
     autofix_prs: false
 repos:
@@ -34,9 +44,11 @@ repos:
     -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
-        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
+        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg,
+                 post-checkout, post-commit, post-merge, post-rewrite]
     -   id: trailing-whitespace
-        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
+        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg,
+                 post-checkout, post-commit, post-merge, post-rewrite]
 -   repo: https://github.com/cpplint/cpplint
     rev: 1.6.1
     hooks:
@@ -46,7 +58,13 @@ repos:
         # this particular codebase (e.g. src/headers, src/klib). However,
         # we can lint all header files since they aren't "generated" like C files are.
         exclude: ^pandas/_libs/src/(klib|headers)/
-        args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
+        args: [
+            --quiet,
+            '--extensions=c,h',
+            '--headers=h',
+            --recursive,
+            '--filter=-readability/casting,-runtime/int,-build/include_subdir'
+        ]
 -   repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
@@ -107,6 +125,7 @@ repos:
     hooks:
     -   id: yesqa
         additional_dependencies: *flake8_dependencies
+        stages: [manual]
 -   repo: local
     hooks:
     # NOTE: we make `black` a local hook because if it's installed from
@@ -214,7 +233,6 @@ repos:
         exclude: ^pandas/tests/extension/base/base\.py
     -   id: pip-to-conda
         name: Generate pip dependency from conda
-        description: This hook checks if the conda environment.yml and requirements-dev.txt are equal
         language: python
         entry: python scripts/generate_pip_deps_from_conda.py
         files: ^(environment.yml|requirements-dev.txt)$
@@ -311,6 +329,7 @@ repos:
         files: ^pandas
         exclude: ^(pandas/tests|pandas/_version.py|pandas/io/clipboard)
         language: python
+        stages: [manual]
         additional_dependencies:
         - autotyping==22.9.0
         - libcst==0.4.7

asv_bench/benchmarks/io/excel.py

@@ -42,9 +42,8 @@ def setup(self, engine):
     def time_write_excel(self, engine):
         bio = BytesIO()
         bio.seek(0)
-        writer = ExcelWriter(bio, engine=engine)
-        self.df.to_excel(writer, sheet_name="Sheet1")
-        writer.save()
+        with ExcelWriter(bio, engine=engine) as writer:
+            self.df.to_excel(writer, sheet_name="Sheet1")
 
 
 class WriteExcelStyled:
@@ -57,13 +56,12 @@ def setup(self, engine):
     def time_write_excel_style(self, engine):
         bio = BytesIO()
         bio.seek(0)
-        writer = ExcelWriter(bio, engine=engine)
-        df_style = self.df.style
-        df_style.applymap(lambda x: "border: red 1px solid;")
-        df_style.applymap(lambda x: "color: blue")
-        df_style.applymap(lambda x: "border-color: green black", subset=["float1"])
-        df_style.to_excel(writer, sheet_name="Sheet1")
-        writer.save()
+        with ExcelWriter(bio, engine=engine) as writer:
+            df_style = self.df.style
+            df_style.applymap(lambda x: "border: red 1px solid;")
+            df_style.applymap(lambda x: "color: blue")
+            df_style.applymap(lambda x: "border-color: green black", subset=["float1"])
+            df_style.to_excel(writer, sheet_name="Sheet1")
 
 
 class ReadExcel:

ci/deps/actions-38-downstream_compat.yaml

@@ -34,12 +34,11 @@ dependencies:
   - gcsfs
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib>=3.6.1
   - numba
   - numexpr
   - openpyxl
   - odfpy
-  - pandas-gbq
   - psycopg2
   - pyarrow<10
   - pymysql
@@ -68,5 +67,6 @@ dependencies:
   - statsmodels
   - coverage
   - pandas-datareader
+  - pandas-gbq
   - pyyaml
   - py

ci/deps/circle-38-arm64.yaml

@@ -33,7 +33,7 @@ dependencies:
   - gcsfs
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib>=3.6.1
   - numba
   - numexpr
   - openpyxl

doc/source/development/debugging_extensions.rst

@@ -119,3 +119,13 @@ Note that code execution under valgrind will take much longer than usual. While
 .. note::
 
    For best results, you should run use a Python installation configured with Valgrind support (--with-valgrind)
+
+
+Easier code navigation
+======================
+
+Generating a ``compile_commands.json`` file may make it easier to navigate the C extensions, as this allows your code editor to list references, jump to definitions, etc... To make this work with setuptools you can use `Bear <https://github.com/rizsotto/Bear>`_.
+
+.. code-block:: sh
+
+   bear -- python setup.py build_ext --inplace -j4 --with-debugging-symbols

doc/source/reference/extensions.rst

@@ -32,6 +32,7 @@ objects.
    .. autosummary::
       :toctree: api/
 
+      api.extensions.ExtensionArray._accumulate
       api.extensions.ExtensionArray._concat_same_type
       api.extensions.ExtensionArray._formatter
       api.extensions.ExtensionArray._from_factorized

doc/source/user_guide/basics.rst

@@ -827,20 +827,54 @@ In this case, provide ``pipe`` with a tuple of ``(callable, data_keyword)``.
 
 For example, we can fit a regression using statsmodels. Their API expects a formula first and a ``DataFrame`` as the second argument, ``data``. We pass in the function, keyword pair ``(sm.ols, 'data')`` to ``pipe``:
 
-.. ipython:: python
-   :okwarning:
-
-   import statsmodels.formula.api as sm
-
-   bb = pd.read_csv("data/baseball.csv", index_col="id")
+.. code-block:: ipython
 
-   (
-       bb.query("h > 0")
-       .assign(ln_h=lambda df: np.log(df.h))
-       .pipe((sm.ols, "data"), "hr ~ ln_h + year + g + C(lg)")
-       .fit()
-       .summary()
-   )
+   In [147]: import statsmodels.formula.api as sm
+
+   In [148]: bb = pd.read_csv("data/baseball.csv", index_col="id")
+
+   In [149]: (
+      .....:     bb.query("h > 0")
+      .....:     .assign(ln_h=lambda df: np.log(df.h))
+      .....:     .pipe((sm.ols, "data"), "hr ~ ln_h + year + g + C(lg)")
+      .....:     .fit()
+      .....:     .summary()
+      .....: )
+      .....:
+   Out[149]:
+   <class 'statsmodels.iolib.summary.Summary'>
+   """
+                              OLS Regression Results
+   ==============================================================================
+   Dep. Variable:                     hr   R-squared:                       0.685
+   Model:                            OLS   Adj. R-squared:                  0.665
+   Method:                 Least Squares   F-statistic:                     34.28
+   Date:                Tue, 22 Nov 2022   Prob (F-statistic):           3.48e-15
+   Time:                        05:34:17   Log-Likelihood:                -205.92
+   No. Observations:                  68   AIC:                             421.8
+   Df Residuals:                      63   BIC:                             432.9
+   Df Model:                           4
+   Covariance Type:            nonrobust
+   ===============================================================================
+                     coef    std err          t      P>|t|      [0.025      0.975]
+   -------------------------------------------------------------------------------
+   Intercept   -8484.7720   4664.146     -1.819      0.074   -1.78e+04     835.780
+   C(lg)[T.NL]    -2.2736      1.325     -1.716      0.091      -4.922       0.375
+   ln_h           -1.3542      0.875     -1.547      0.127      -3.103       0.395
+   year            4.2277      2.324      1.819      0.074      -0.417       8.872
+   g               0.1841      0.029      6.258      0.000       0.125       0.243
+   ==============================================================================
+   Omnibus:                       10.875   Durbin-Watson:                   1.999
+   Prob(Omnibus):                  0.004   Jarque-Bera (JB):               17.298
+   Skew:                           0.537   Prob(JB):                     0.000175
+   Kurtosis:                       5.225   Cond. No.                     1.49e+07
+   ==============================================================================
+
+   Notes:
+   [1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
+   [2] The condition number is large, 1.49e+07. This might indicate that there are
+   strong multicollinearity or other numerical problems.
+   """
 
 The pipe method is inspired by unix pipes and more recently dplyr_ and magrittr_, which
 have introduced the popular ``(%>%)`` (read pipe) operator for R_.
@@ -1213,12 +1247,6 @@ With a DataFrame, you can simultaneously reindex the index and columns:
    df
    df.reindex(index=["c", "f", "b"], columns=["three", "two", "one"])
 
-You may also use ``reindex`` with an ``axis`` keyword:
-
-.. ipython:: python
-
-   df.reindex(["c", "f", "b"], axis="index")
-
 Note that the ``Index`` objects containing the actual axis labels can be
 **shared** between objects. So if we have a Series and a DataFrame, the
 following can be done:

doc/source/whatsnew/v0.16.2.rst

@@ -61,21 +61,55 @@ In the example above, the functions ``f``, ``g``, and ``h`` each expected the Da
 When the function you wish to apply takes its data anywhere other than the first argument, pass a tuple
 of ``(function, keyword)`` indicating where the DataFrame should flow. For example:
 
-.. ipython:: python
-   :okwarning:
-
-   import statsmodels.formula.api as sm
-
-   bb = pd.read_csv("data/baseball.csv", index_col="id")
-
-   # sm.ols takes (formula, data)
-   (
-       bb.query("h > 0")
-       .assign(ln_h=lambda df: np.log(df.h))
-       .pipe((sm.ols, "data"), "hr ~ ln_h + year + g + C(lg)")
-       .fit()
-       .summary()
-   )
+.. code-block:: ipython
+
+    In [1]: import statsmodels.formula.api as sm
+
+    In [2]: bb = pd.read_csv("data/baseball.csv", index_col="id")
+
+    # sm.ols takes (formula, data)
+    In [3]: (
+    ...:     bb.query("h > 0")
+    ...:     .assign(ln_h=lambda df: np.log(df.h))
+    ...:     .pipe((sm.ols, "data"), "hr ~ ln_h + year + g + C(lg)")
+    ...:     .fit()
+    ...:     .summary()
+    ...: )
+    ...:
+    Out[3]:
+    <class 'statsmodels.iolib.summary.Summary'>
+    """
+                                OLS Regression Results
+    ==============================================================================
+    Dep. Variable:                     hr   R-squared:                       0.685
+    Model:                            OLS   Adj. R-squared:                  0.665
+    Method:                 Least Squares   F-statistic:                     34.28
+    Date:                Tue, 22 Nov 2022   Prob (F-statistic):           3.48e-15
+    Time:                        05:35:23   Log-Likelihood:                -205.92
+    No. Observations:                  68   AIC:                             421.8
+    Df Residuals:                      63   BIC:                             432.9
+    Df Model:                           4
+    Covariance Type:            nonrobust
+    ===============================================================================
+                    coef    std err          t      P>|t|      [0.025      0.975]
+    -------------------------------------------------------------------------------
+    Intercept   -8484.7720   4664.146     -1.819      0.074   -1.78e+04     835.780
+    C(lg)[T.NL]    -2.2736      1.325     -1.716      0.091      -4.922       0.375
+    ln_h           -1.3542      0.875     -1.547      0.127      -3.103       0.395
+    year            4.2277      2.324      1.819      0.074      -0.417       8.872
+    g               0.1841      0.029      6.258      0.000       0.125       0.243
+    ==============================================================================
+    Omnibus:                       10.875   Durbin-Watson:                   1.999
+    Prob(Omnibus):                  0.004   Jarque-Bera (JB):               17.298
+    Skew:                           0.537   Prob(JB):                     0.000175
+    Kurtosis:                       5.225   Cond. No.                     1.49e+07
+    ==============================================================================
+
+    Notes:
+    [1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
+    [2] The condition number is large, 1.49e+07. This might indicate that there are
+    strong multicollinearity or other numerical problems.
+    """
 
 The pipe method is inspired by unix pipes, which stream text through
 processes. More recently dplyr_ and magrittr_ have introduced the

doc/source/whatsnew/v1.5.3.rst

@@ -27,6 +27,8 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 - Bug in :meth:`.Styler.to_excel` leading to error when unrecognized ``border-style`` (e.g. ``"hair"``) provided to Excel writers (:issue:`48649`)
+- Bug when chaining several :meth:`.Styler.concat` calls, only the last styler was concatenated (:issue:`49207`)
+- Fixed bug when instantiating a :class:`DataFrame` subclass inheriting from ``typing.Generic`` that triggered a ``UserWarning`` on python 3.11 (:issue:`49649`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -35,7 +37,6 @@ Bug fixes
 Other
 ~~~~~
 -
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_153.contributors: