Merge remote-tracking branch 'upstream/main' into pr/Anupam-USP-deprecate_stacklevel

Author: MarcoGorelli

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,5 @@
 - [ ] closes #xxxx (Replace xxxx with the Github issue number)
 - [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
 - [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
+- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
 - [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.

.github/actions/setup-conda/action.yml

@@ -3,6 +3,12 @@ inputs:
   environment-file:
     description: Conda environment file to use.
     default: environment.yml
+  environment-name:
+    description: Name to use for the Conda environment
+    default: test
+  python-version:
+    description: Python version to install
+    required: false
   pyarrow-version:
     description: If set, overrides the PyArrow version in the Conda environment to the given string.
     required: false
@@ -18,10 +24,14 @@ runs:
       if: ${{ inputs.pyarrow-version }}
 
     - name: Install ${{ inputs.environment-file }}
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v2.1.1
       with:
         environment-file: ${{ inputs.environment-file }}
+        activate-environment: ${{ inputs.environment-name }}
+        python-version: ${{ inputs.python-version }}
         channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
         channels: conda-forge
-        mamba-version: "0.23"
+        mamba-version: "0.24"
         use-mamba: true
+        use-only-tar-bz2: true
+        condarc-file: ci/condarc.yml

.github/workflows/asv-bot.yml

@@ -41,12 +41,8 @@ jobs:
 
         # Although asv sets up its own env, deps are still needed
         # during discovery process
-      - uses: conda-incubator/[email protected]
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      - name: Set up Conda
+        uses: ./.github/actions/setup-conda
 
       - name: Run benchmarks
         id: bench

.github/workflows/code-checks.yml

@@ -58,14 +58,8 @@ jobs:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - uses: conda-incubator/[email protected]
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
       id: build
@@ -127,14 +121,8 @@ jobs:
         path: ~/conda_pkgs_dir
         key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - uses: conda-incubator/[email protected]
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
       id: build

.github/workflows/posix.yml

@@ -147,18 +147,11 @@ jobs:
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }}
 
-    - uses: conda-incubator/[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
-
-    - name: Upgrade Arrow version
-      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
-      if: ${{ matrix.pyarrow_version }}
+        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

.github/workflows/sdist.yml

@@ -59,11 +59,12 @@ jobs:
         name: ${{matrix.python-version}}-sdist.gz
         path: dist/*.gz
 
-    - uses: conda-incubator/[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        activate-environment: pandas-sdist
-        channels: conda-forge
-        python-version: '${{ matrix.python-version }}'
+        environment-file: ""
+        environment-name: pandas-sdist
+        python-version: ${{ matrix.python-version }}
 
     - name: Install pandas from sdist
       run: |
@@ -74,9 +75,9 @@ jobs:
       run: |
         case "${{matrix.python-version}}" in
         3.8)
-          pip install numpy==1.18.5 ;;
+          pip install numpy==1.19.5 ;;
         3.9)
-          pip install numpy==1.19.3 ;;
+          pip install numpy==1.19.5 ;;
         3.10)
           pip install numpy==1.21.2 ;;
         esac

.pre-commit-config.yaml

@@ -26,7 +26,6 @@ repos:
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
-        files: ^(pandas|doc)/
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
     hooks:
@@ -93,7 +92,19 @@ repos:
         pass_filenames: false
         types: [python]
         stages: [manual]
-        additional_dependencies: ['[email protected]']
+        additional_dependencies: &pyright_dependencies
+        - [email protected]
+-   repo: local
+    hooks:
+    -   id: pyright_reportGeneralTypeIssues
+        name: pyright reportGeneralTypeIssues
+        entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json
+        # note: assumes python env is setup and activated
+        language: node
+        pass_filenames: false
+        types: [python]
+        stages: [manual]
+        additional_dependencies: *pyright_dependencies
 -   repo: local
     hooks:
     -   id: mypy

asv_bench/benchmarks/frame_ctor.py

@@ -37,7 +37,7 @@ def setup(self):
         self.dict_list = frame.to_dict(orient="records")
         self.data2 = {i: {j: float(j) for j in range(100)} for i in range(2000)}
 
-        # arrays which we wont consolidate
+        # arrays which we won't consolidate
         self.dict_of_categoricals = {i: Categorical(np.arange(N)) for i in range(K)}
 
     def time_list_of_dict(self):
@@ -60,7 +60,7 @@ def time_nested_dict_int64(self):
         DataFrame(self.data2)
 
     def time_dict_of_categoricals(self):
-        # dict of arrays that we wont consolidate
+        # dict of arrays that we won't consolidate
         DataFrame(self.dict_of_categoricals)
 
 

asv_bench/benchmarks/frame_methods.py

@@ -288,6 +288,26 @@ def time_values_mixed_wide(self):
         self.df_mixed_wide.values
 
 
+class ToRecords:
+    def setup(self):
+        N = 100_000
+        data = np.random.randn(N, 2)
+        mi = MultiIndex.from_arrays(
+            [
+                np.arange(N),
+                date_range("1970-01-01", periods=N, freq="ms"),
+            ]
+        )
+        self.df = DataFrame(data)
+        self.df_mi = DataFrame(data, index=mi)
+
+    def time_to_records(self):
+        self.df.to_records(index=True)
+
+    def time_to_records_multiindex(self):
+        self.df_mi.to_records(index=True)
+
+
 class Repr:
     def setup(self):
         nrows = 10000

asv_bench/benchmarks/groupby.py

@@ -527,7 +527,7 @@ def time_dtype_as_field(self, dtype, method, application, ncols):
 
 class GroupByCythonAgg:
     """
-    Benchmarks specifically targetting our cython aggregation algorithms
+    Benchmarks specifically targeting our cython aggregation algorithms
     (using a big enough dataframe with simple key, so a large part of the
     time is actually spent in the grouped aggregation).
     """

asv_bench/benchmarks/io/excel.py

@@ -86,4 +86,15 @@ def time_read_excel(self, engine):
         read_excel(fname, engine=engine)
 
 
+class ReadExcelNRows(ReadExcel):
+    def time_read_excel(self, engine):
+        if engine == "xlrd":
+            fname = self.fname_excel_xls
+        elif engine == "odf":
+            fname = self.fname_odf
+        else:
+            fname = self.fname_excel
+        read_excel(fname, engine=engine, nrows=10)
+
+
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/libs.py

@@ -2,7 +2,7 @@
 Benchmarks for code in pandas/_libs, excluding pandas/_libs/tslibs,
 which has its own directory.
 
-If a PR does not edit anything in _libs/, then it is unlikely that thes
+If a PR does not edit anything in _libs/, then it is unlikely that the
 benchmarks will be affected.
 """
 import numpy as np

asv_bench/benchmarks/replace.py

@@ -50,7 +50,7 @@ def time_replace_list(self, inplace):
         self.df.replace([np.inf, -np.inf], np.nan, inplace=inplace)
 
     def time_replace_list_one_match(self, inplace):
-        # the 1 can be held in self._df.blocks[0], while the inf and -inf cant
+        # the 1 can be held in self._df.blocks[0], while the inf and -inf can't
         self.df.replace([np.inf, -np.inf, 1], np.nan, inplace=inplace)
 
 

asv_bench/benchmarks/reshape.py

@@ -78,7 +78,7 @@ def time_stack(self, dtype):
         self.df.stack()
 
     def time_unstack_fast(self, dtype):
-        # last level -> doesnt have to make copies
+        # last level -> doesn't have to make copies
         self.ser.unstack("bar")
 
     def time_unstack_slow(self, dtype):

asv_bench/benchmarks/tslibs/period.py

@@ -1,6 +1,6 @@
 """
-Period benchmarks that rely only on tslibs.  See benchmarks.period for
-Period benchmarks that rely on other parts fo pandas.
+Period benchmarks that rely only on tslibs. See benchmarks.period for
+Period benchmarks that rely on other parts of pandas.
 """
 
 import numpy as np

asv_bench/benchmarks/tslibs/timedelta.py

@@ -1,6 +1,6 @@
 """
-Timedelta benchmarks that rely only on tslibs.  See benchmarks.timedeltas for
-Timedelta benchmarks that rely on other parts fo pandas.
+Timedelta benchmarks that rely only on tslibs. See benchmarks.timedeltas for
+Timedelta benchmarks that rely on other parts of pandas.
 """
 import datetime
 

ci/condarc.yml

@@ -0,0 +1,32 @@
+# https://docs.conda.io/projects/conda/en/latest/configuration.html
+
+# always_yes (NoneType, bool)
+#   aliases: yes
+#   Automatically choose the 'yes' option whenever asked to proceed with a
+#   conda operation, such as when running `conda install`.
+#
+always_yes: true
+
+# remote_connect_timeout_secs (float)
+#   The number seconds conda will wait for your client to establish a
+#   connection to a remote url resource.
+#
+remote_connect_timeout_secs: 30.0
+
+# remote_max_retries (int)
+#   The maximum number of retries each HTTP connection should attempt.
+#
+remote_max_retries: 10
+
+# remote_backoff_factor (int)
+#   The factor determines the time HTTP connection should wait for
+#   attempt.
+#
+remote_backoff_factor: 3
+
+# remote_read_timeout_secs (float)
+#   Once conda has connected to a remote resource and sent an HTTP
+#   request, the read timeout is the number of seconds conda will wait for
+#   the server to send a response.
+#
+remote_read_timeout_secs: 60.0