Merge branch 'main' into cow_infer_objects

Author: phofl

.github/workflows/python-dev.yml

@@ -73,7 +73,7 @@ jobs:
       run: |
         python --version
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
+        python -m pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
         python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17

.pre-commit-config.yaml

@@ -15,6 +15,10 @@ default_stages: [
 ci:
     autofix_prs: false
 repos:
+-   repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.215
+    hooks:
+    -   id: ruff
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.1
     hooks:
@@ -66,13 +70,6 @@ repos:
             --linelength=88,
             '--filter=-readability/casting,-runtime/int,-build/include_subdir,-readability/fn_size'
         ]
--   repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-    -   id: flake8
-        additional_dependencies: &flake8_dependencies
-        - flake8==6.0.0
-        - flake8-bugbear==22.7.1
 -   repo: https://github.com/pycqa/pylint
     rev: v2.15.9
     hooks:
@@ -117,12 +114,6 @@ repos:
     rev: v0.6.7
     hooks:
     - id: sphinx-lint
--   repo: https://github.com/asottile/yesqa
-    rev: v1.4.0
-    hooks:
-    -   id: yesqa
-        additional_dependencies: *flake8_dependencies
-        stages: [manual]
 -   repo: local
     hooks:
     # NOTE: we make `black` a local hook because if it's installed from
@@ -326,13 +317,6 @@ repos:
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
         additional_dependencies: [pyyaml, toml]
-    -   id: sync-flake8-versions
-        name: Check flake8 version is synced across flake8, yesqa, and environment.yml
-        language: python
-        entry: python scripts/sync_flake8_versions.py
-        files: ^(\.pre-commit-config\.yaml|environment\.yml)$
-        pass_filenames: false
-        additional_dependencies: [pyyaml, toml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py

asv_bench/benchmarks/array.py

@@ -44,6 +44,15 @@ def time_from_integer_array(self):
         pd.array(self.values_integer, dtype="Int64")
 
 
+class IntervalArray:
+    def setup(self):
+        N = 10_000
+        self.tuples = [(i, i + 1) for i in range(N)]
+
+    def time_from_tuples(self):
+        pd.arrays.IntervalArray.from_tuples(self.tuples)
+
+
 class StringArray:
     def setup(self):
         N = 100_000

asv_bench/benchmarks/indexing.py

@@ -476,6 +476,19 @@ def time_assign_list_of_columns_concat(self):
         concat([self.df, df], axis=1)
 
 
+class Setitem:
+    def setup(self):
+        N = 500_000
+        cols = 500
+        self.df = DataFrame(np.random.rand(N, cols))
+
+    def time_setitem(self):
+        self.df[100] = 100
+
+    def time_setitem_list(self):
+        self.df[[100, 200, 300]] = 100
+
+
 class ChainIndexing:
 
     params = [None, "warn"]

ci/code_checks.sh

@@ -83,6 +83,31 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05,SS06
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Partially validate docstrings (RT02)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=RT02 --ignore_functions \
+    	    pandas.Series.align \
+	    pandas.Series.dt.total_seconds \
+	    pandas.Series.cat.rename_categories \
+	    pandas.Series.cat.reorder_categories \
+	    pandas.Series.cat.add_categories \
+	    pandas.Series.cat.remove_categories \
+	    pandas.Series.cat.remove_unused_categories \
+	    pandas.Index.all \
+	    pandas.Index.any \
+	    pandas.MultiIndex.drop \
+	    pandas.DatetimeIndex.to_pydatetime \
+	    pandas.TimedeltaIndex.to_pytimedelta \
+	    pandas.core.groupby.SeriesGroupBy.apply \
+	    pandas.core.groupby.DataFrameGroupBy.apply \
+	    pandas.io.formats.style.Styler.export \
+	    pandas.api.extensions.ExtensionArray.astype \
+	    pandas.api.extensions.ExtensionArray.dropna \
+	    pandas.api.extensions.ExtensionArray.isna \
+	    pandas.api.extensions.ExtensionArray.repeat \
+	    pandas.api.extensions.ExtensionArray.unique \
+	    pandas.DataFrame.align
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### DOCUMENTATION NOTEBOOKS ###

doc/scripts/eval_performance.py

@@ -6,6 +6,7 @@
 from pandas import DataFrame
 
 setup_common = """from pandas import DataFrame
+import numpy as np
 df = DataFrame(np.random.randn(%d, 3), columns=list('abc'))
 %s"""
 

doc/source/development/contributing_codebase.rst

@@ -43,7 +43,7 @@ Pre-commit
 ----------
 
 Additionally, :ref:`Continuous Integration <contributing.ci>` will run code formatting checks
-like ``black``, ``flake8``,
+like ``black``, ``ruff``,
 ``isort``, and ``cpplint`` and more using `pre-commit hooks <https://pre-commit.com/>`_
 Any warnings from these checks will cause the :ref:`Continuous Integration <contributing.ci>` to fail; therefore,
 it is helpful to run the check yourself before submitting code. This
@@ -783,6 +783,7 @@ preferred if the inputs or logic are simple, with Hypothesis tests reserved
 for cases with complex logic or where there are too many combinations of
 options or subtle interactions to test (or think of!) all of them.
 
+.. _contributing.running_tests:
 
 Running the test suite
 ----------------------

doc/source/development/contributing_documentation.rst

@@ -127,6 +127,7 @@ for some tips and tricks to get the doctests passing.
 When doing a PR with a docstring update, it is good to post the
 output of the validation script in a comment on github.
 
+.. _contributing.howto-build-docs:
 
 How to build the pandas documentation
 ---------------------------------------

doc/source/development/contributing_environment.rst

@@ -12,8 +12,11 @@ changes, you can skip to :ref:`contributing to the documentation <contributing_d
 creating the development environment you won't be able to build the documentation
 locally before pushing your changes. It's recommended to also install the :ref:`pre-commit hooks <contributing.pre-commit>`.
 
-.. contents:: Table of contents:
-   :local:
+.. toctree::
+    :maxdepth: 2
+    :hidden:
+
+    contributing_gitpod.rst
 
 Step 1: install a C compiler
 ----------------------------
@@ -76,7 +79,7 @@ Step 2: create an isolated environment
 Before we begin, please:
 
 * Make sure that you have :any:`cloned the repository <contributing.forking>`
-* ``cd`` to the pandas source directory
+* ``cd`` to the pandas source directory you just created with the clone command
 
 .. _contributing.mamba:
 
@@ -188,6 +191,17 @@ Enable Docker support and use the Services tool window to build and manage image
 run and interact with containers.
 See https://www.jetbrains.com/help/pycharm/docker.html for details.
 
+Option 4: using Gitpod
+~~~~~~~~~~~~~~~~~~~~~~
+
+Gitpod is an open-source platform that automatically creates the correct development
+environment right in your browser, reducing the need to install local development
+environments and deal with incompatible dependencies.
+
+If you are a Windows user, unfamiliar with using the command line or building pandas
+for the first time, it is often faster to build with Gitpod. Here are the in-depth instructions
+for :ref:`building pandas with GitPod <contributing-gitpod>`.
+
 Step 3: build and install pandas
 --------------------------------