Merge branch 'master' into stringarray-nan

Author: lithomas1

.github/workflows/ci.yml

@@ -41,6 +41,8 @@ jobs:
 
     - uses: conda-incubator/setup-miniconda@v2
       with:
+        mamba-version: "*"
+        channels: conda-forge
         activate-environment: pandas-dev
         channel-priority: strict
         environment-file: ${{ env.ENV_FILE }}
@@ -104,6 +106,8 @@ jobs:
 
     - uses: conda-incubator/setup-miniconda@v2
       with:
+        mamba-version: "*"
+        channels: conda-forge
         activate-environment: pandas-dev
         channel-priority: strict
         environment-file: ${{ env.ENV_FILE }}

.github/workflows/database.yml

@@ -88,6 +88,8 @@ jobs:
 
     - uses: conda-incubator/setup-miniconda@v2
       with:
+        mamba-version: "*"
+        channels: conda-forge
         activate-environment: pandas-dev
         channel-priority: strict
         environment-file: ${{ matrix.ENV_FILE }}

.github/workflows/posix.yml

@@ -81,6 +81,8 @@ jobs:
 
     - uses: conda-incubator/setup-miniconda@v2
       with:
+        mamba-version: "*"
+        channels: conda-forge
         activate-environment: pandas-dev
         channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}

.github/workflows/python-dev.yml

@@ -45,10 +45,11 @@ jobs:
       with:
         python-version: '3.10-dev'
 
+    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install dependencies
       shell: bash
       run: |
-        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install --upgrade pip "setuptools<60.0.0" wheel
         pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/nedbat/coveragepy.git
         pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov

.github/workflows/sdist.yml

@@ -39,9 +39,10 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install --upgrade pip "setuptools<60.0.0" wheel
 
         # GH 39416
         pip install numpy
@@ -57,8 +58,10 @@ jobs:
         channels: conda-forge
         python-version: '${{ matrix.python-version }}'
 
+    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install pandas from sdist
       run: |
+        python -m pip install --upgrade "setuptools<60.0.0"
         pip list
         python -m pip install dist/*.gz
 

MANIFEST.in

@@ -36,6 +36,7 @@ global-exclude *.xpt
 global-exclude *.cpt
 global-exclude *.xz
 global-exclude *.zip
+global-exclude *.zst
 global-exclude *~
 global-exclude .DS_Store
 global-exclude .git*

asv_bench/benchmarks/frame_ctor.py

@@ -182,4 +182,21 @@ def time_frame_from_arrays_sparse(self):
         )
 
 
+class From3rdParty:
+    # GH#44616
+
+    def setup(self):
+        try:
+            import torch
+        except ImportError:
+            raise NotImplementedError
+
+        row = 700000
+        col = 64
+        self.val_tensor = torch.randn(row, col)
+
+    def time_from_torch(self):
+        DataFrame(self.val_tensor)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

azure-pipelines.yml

@@ -36,13 +36,14 @@ jobs:
     vmImage: ubuntu-18.04
 
   steps:
+    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - script: |
         docker pull quay.io/pypa/manylinux2014_i686
         docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
         /bin/bash -xc "cd pandas && \
         /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
-        python -m pip install --no-deps -U pip wheel setuptools && \
+        python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
         pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \

ci/azure/posix.yml

@@ -8,17 +8,16 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      ${{ if eq(parameters.name, 'macOS') }}:
-        py38_macos_1:
-          ENV_FILE: ci/deps/azure-macos-38.yaml
-          CONDA_PY: "38"
-          PATTERN: "not slow"
-          PYTEST_TARGET: "pandas/tests/[a-h]*"
-        py38_macos_2:
-          ENV_FILE: ci/deps/azure-macos-38.yaml
-          CONDA_PY: "38"
-          PATTERN: "not slow"
-          PYTEST_TARGET: "pandas/tests/[i-z]*"
+      py38_macos_1:
+        ENV_FILE: ci/deps/azure-macos-38.yaml
+        CONDA_PY: "38"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+      py38_macos_2:
+        ENV_FILE: ci/deps/azure-macos-38.yaml
+        CONDA_PY: "38"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:
     - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'

ci/code_checks.sh

@@ -66,22 +66,22 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Doctests' ; echo $MSG
     python -m pytest --doctest-modules \
+      pandas/_config/ \
       pandas/_libs/ \
+      pandas/_testing/ \
       pandas/api/ \
       pandas/arrays/ \
       pandas/compat/ \
       pandas/core \
       pandas/errors/ \
-      pandas/io/clipboard/ \
-      pandas/io/json/ \
-      pandas/io/excel/ \
-      pandas/io/parsers/ \
-      pandas/io/sas/ \
-      pandas/io/sql.py \
-      pandas/io/formats/format.py \
-      pandas/io/formats/style.py \
-      pandas/io/stata.py \
-      pandas/tseries/
+      pandas/io/ \
+      pandas/plotting/ \
+      pandas/tseries/ \
+      pandas/util/ \
+      pandas/_typing.py \
+      pandas/_version.py \
+      pandas/conftest.py \
+      pandas/testing.py
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Cython Doctests' ; echo $MSG

ci/deps/actions-38-db.yaml

@@ -3,6 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - python=3.8
+  - pip
 
   # tools
   - cython>=0.29.24
@@ -45,9 +46,9 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - pip
+  - brotlipy
+  - coverage
+  - pandas-datareader
+  - pyxlsb
   - pip:
-    - brotlipy
-    - coverage
-    - pandas-datareader
-    - pyxlsb
+    - torch

ci/deps/actions-38-locale.yaml

@@ -33,6 +33,4 @@ dependencies:
   - xlsxwriter
   - xlwt
   - pyarrow=1.0.1
-  - pip
-  - pip:
-    - pyxlsb
+  - pyxlsb

ci/deps/actions-38-slow.yaml

@@ -34,3 +34,4 @@ dependencies:
   - xlsxwriter
   - xlwt
   - numba
+  - zstandard

ci/deps/actions-39-slow.yaml

@@ -26,7 +26,7 @@ dependencies:
   - numexpr
   - numpy
   - openpyxl
-  - pyarrow
+  - pyarrow>2.0.1
   - pytables
   - python-dateutil
   - pytz
@@ -37,6 +37,5 @@ dependencies:
   - xlsxwriter
   - xlwt
   - pyreadstat
-  - pip
-  - pip:
-    - pyxlsb
+  - pyxlsb
+  - zstandard

ci/deps/actions-39.yaml

@@ -21,11 +21,11 @@ dependencies:
   - jinja2
   - lxml
   - matplotlib
+  - numpy  # move up to solve before numba
   - numba
   - numexpr
-  - numpy
   - openpyxl
-  - pyarrow
+  - pyarrow>2.0.1
   - pytables
   - python-dateutil
   - pytz
@@ -36,6 +36,5 @@ dependencies:
   - xlsxwriter
   - xlwt
   - pyreadstat
-  - pip
-  - pip:
-    - pyxlsb
+  - pyxlsb
+  - zstandard

ci/deps/azure-macos-38.yaml

@@ -23,15 +23,16 @@ dependencies:
   - numpy=1.18.5
   - openpyxl
   - pyarrow=1.0.1
+  - pyreadstat
   - pytables
   - python-dateutil==2.8.1
   - pytz
+  - pyxlsb
   - xarray
   - xlrd
   - xlsxwriter
   - xlwt
+  - zstandard
   - pip
   - pip:
     - cython>=0.29.24
-    - pyreadstat
-    - pyxlsb

ci/deps/azure-windows-38.yaml

@@ -32,3 +32,4 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
+  - zstandard

ci/deps/azure-windows-39.yaml

@@ -36,6 +36,5 @@ dependencies:
   - xlsxwriter
   - xlwt
   - pyreadstat
-  - pip
-  - pip:
-    - pyxlsb
+  - pyxlsb
+  - zstandard

ci/deps/circle-38-arm64.yaml

@@ -12,10 +12,10 @@ dependencies:
 
   # pandas dependencies
   - botocore>=1.11
+  - flask
+  - moto
   - numpy
   - python-dateutil
   - pytz
+  - zstandard
   - pip
-  - flask
-  - pip:
-    - moto

ci/setup_env.sh

@@ -48,6 +48,7 @@ conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
 conda install pip conda  # create conda to create a historical artifact for pip & setuptools
 conda update -n base conda
+conda install -y -c conda-forge mamba
 
 echo "conda info -a"
 conda info -a
@@ -62,8 +63,8 @@ conda list
 conda remove --all -q -y -n pandas-dev
 
 echo
-echo "conda env create -q --file=${ENV_FILE}"
-time conda env create -q --file="${ENV_FILE}"
+echo "mamba env create -q --file=${ENV_FILE}"
+time mamba env create -q --file="${ENV_FILE}"
 
 
 if [[ "$BITS32" == "yes" ]]; then
@@ -86,11 +87,6 @@ echo "w/o removing anything else"
 conda remove pandas -y --force || true
 pip uninstall -y pandas || true
 
-echo
-echo "remove postgres if has been installed with conda"
-echo "we use the one from the CI"
-conda remove postgresql -y --force || true
-
 echo
 echo "remove qt"
 echo "causes problems with the clipboard, we use xsel for that"
@@ -106,7 +102,8 @@ echo "[Build extensions]"
 python setup.py build_ext -q -j2
 
 echo "[Updating pip]"
-python -m pip install --no-deps -U pip wheel setuptools
+# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
+python -m pip install --no-deps -U pip wheel "setuptools<60.0.0"
 
 echo "[Install pandas]"
 python -m pip install --no-build-isolation -e .
@@ -115,13 +112,4 @@ echo
 echo "conda list"
 conda list
 
-# Install DB for Linux
-
-if [[ -n ${SQL:0} ]]; then
-  echo "installing dbs"
-  mysql -e 'create database pandas_nosetest;'
-  psql -c 'create database pandas_nosetest;' -U postgres
-else
-   echo "not using dbs on non-linux Travis builds or Azure Pipelines"
-fi
 echo "done"