Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

.github/workflows/code-checks.yml

@@ -156,3 +156,27 @@ jobs:
         name: Benchmarks log
         path: asv_bench/benchmarks.log
       if: failure()
+
+  build_docker_dev_environment:
+    name: Build Docker Dev Environment
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-build_docker_dev_environment
+      cancel-in-progress: true
+
+    steps:
+      - name: Clean up dangling images
+        run: docker image prune -f
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Build image
+        run: docker build --pull --no-cache --tag pandas-dev-env .

.github/workflows/datamanger.yml

@@ -20,6 +20,7 @@ jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     services:
       moto:
         image: motoserver/moto
@@ -45,7 +46,7 @@ jobs:
     - name: Run tests
       env:
         PANDAS_DATA_MANAGER: array
-        PATTERN: "not network and not clipboard"
+        PATTERN: "not network and not clipboard and not single_cpu"
         PYTEST_WORKERS: "auto"
         PYTEST_TARGET: pandas
       run: |

.github/workflows/posix.yml

@@ -13,7 +13,6 @@ on:
       - "doc/**"
 
 env:
-  PYTEST_WORKERS: "auto"
   PANDAS_CI: 1
 
 jobs:
@@ -22,35 +21,51 @@ jobs:
     defaults:
       run:
         shell: bash -l {0}
+    timeout-minutes: 120
     strategy:
       matrix:
-        settings: [
-          [actions-38-downstream_compat.yaml, "not slow and not network", "", "", "", "", ""],
-          [actions-38-minimum_versions.yaml, "", "", "", "", "", ""],
-          [actions-38.yaml, "not slow and not network", "language-pack-it", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network", "language-pack-zh-hans", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-38.yaml, "", "", "", "", "", ""],
-          [actions-pypy-38.yaml, "not slow", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39.yaml, "", "", "", "", "", ""],
-          [actions-310-numpydev.yaml, "not slow and not network", "", "", "", "deprecate", "-W error"],
-          [actions-310.yaml, "", "", "", "", "", ""],
-        ]
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+        pattern: ["not single_cpu", "single_cpu"]
+        include:
+          - env_file: actions-38-downstream_compat.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pytest_target: "pandas/tests/test_downstream.py"
+          - env_file: actions-38-minimum_versions.yaml
+            pattern: "not slow and not network and not single_cpu"
+          - env_file: actions-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            extra_apt: "language-pack-it"
+            lang: "it_IT.utf8"
+            lc_all: "it_IT.utf8"
+          - env_file: actions-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            extra_apt: "language-pack-zh-hans"
+            lang: "zh_CN.utf8"
+            lc_all: "zh_CN.utf8"
+          - env_file: actions-pypy-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            test_args: "--max-worker-restart 0"
+          - env_file: actions-310-numpydev.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_testing_mode: "deprecate"
+            test_args: "-W error"
       fail-fast: false
     env:
-      ENV_FILE: ci/deps/${{ matrix.settings[0] }}
-      PATTERN: ${{ matrix.settings[1] }}
-      EXTRA_APT: ${{ matrix.settings[2] }}
-      LANG: ${{ matrix.settings[3] }}
-      LC_ALL: ${{ matrix.settings[4] }}
-      PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
-      TEST_ARGS: ${{ matrix.settings[6] }}
-      PYTEST_TARGET:  pandas
-      IS_PYPY: ${{ contains(matrix.settings[0], 'pypy') }}
+      ENV_FILE: ci/deps/${{ matrix.env_file }}
+      PATTERN: ${{ matrix.pattern }}
+      EXTRA_APT: ${{ matrix.extra_apt || '' }}
+      LANG: ${{ matrix.lang || '' }}
+      LC_ALL: ${{ matrix.lc_all || '' }}
+      PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }}
+      TEST_ARGS: ${{ matrix.test_args || '' }}
+      PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
+      PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }}
       # TODO: re-enable coverage on pypy, its slow
-      COVERAGE: ${{ !contains(matrix.settings[0], 'pypy') }}
+      COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}-${{ matrix.settings[1] }}-${{ matrix.settings[2] }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}
       cancel-in-progress: true
 
     services:
@@ -128,8 +143,7 @@ jobs:
       shell: bash
       run: |
         # TODO: re-enable cov, its slowing the tests down though
-        # TODO: Unpin Cython, the new Cython 0.29.26 is causing compilation errors
-        pip install Cython==0.29.25 numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
+        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
       if: ${{ env.IS_PYPY == 'true' }}
 
     - name: Build Pandas

.github/workflows/python-dev.yml

@@ -23,7 +23,7 @@ on:
 env:
   PYTEST_WORKERS: "auto"
   PANDAS_CI: 1
-  PATTERN: "not slow and not network and not clipboard"
+  PATTERN: "not slow and not network and not clipboard and not single_cpu"
   COVERAGE: true
   PYTEST_TARGET: pandas
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/condaforge/miniforge3
+FROM quay.io/condaforge/miniforge3:4.11.0-0
 
 # if you forked pandas, you can pass in your own GitHub username to use your fork
 # i.e. gh_username=myname
@@ -45,4 +45,4 @@ RUN . /opt/conda/etc/profile.d/conda.sh \
     && cd "$pandas_home" \
     && export \
     && python setup.py build_ext -j 4 \
-    && python -m pip install -e .
+    && python -m pip install --no-build-isolation -e .

README.md

@@ -136,7 +136,7 @@ or alternatively
 python setup.py develop
 ```
 
-See the full instructions for [installing from source](https://pandas.pydata.org/pandas-docs/stable/install.html#installing-from-source).
+See the full instructions for [installing from source](https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#installing-from-source).
 
 ## License
 [BSD 3](LICENSE)

asv_bench/benchmarks/multiindex_object.py

@@ -47,6 +47,29 @@ def time_small_get_loc_warm(self):
             self.mi_small.get_loc((99, "A", "A"))
 
 
+class GetLocs:
+    def setup(self):
+        self.mi_large = MultiIndex.from_product(
+            [np.arange(1000), np.arange(20), list(string.ascii_letters)],
+            names=["one", "two", "three"],
+        )
+        self.mi_med = MultiIndex.from_product(
+            [np.arange(1000), np.arange(10), list("A")], names=["one", "two", "three"]
+        )
+        self.mi_small = MultiIndex.from_product(
+            [np.arange(100), list("A"), list("A")], names=["one", "two", "three"]
+        )
+
+    def time_large_get_locs(self):
+        self.mi_large.get_locs([999, 19, "Z"])
+
+    def time_med_get_locs(self):
+        self.mi_med.get_locs([999, 9, "A"])
+
+    def time_small_get_locs(self):
+        self.mi_small.get_locs([99, "A", "A"])
+
+
 class Duplicates:
     def setup(self):
         size = 65536

azure-pipelines.yml

@@ -18,7 +18,7 @@ pr:
 variables:
   PYTEST_WORKERS: auto
   PYTEST_TARGET:  pandas
-  PATTERN: "not slow and not high_memory and not db and not network"
+  PATTERN: "not slow and not high_memory and not db and not network and not single_cpu"
   PANDAS_CI: 1
 
 jobs:

ci/azure/posix.yml

@@ -4,40 +4,22 @@ parameters:
 
 jobs:
 - job: ${{ parameters.name }}
+  timeoutInMinutes: 90
   pool:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py38_macos_1:
+      py38:
         ENV_FILE: ci/deps/azure-macos-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow"
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
-      py38_macos_2:
-        ENV_FILE: ci/deps/azure-macos-38.yaml
-        CONDA_PY: "38"
-        PATTERN: "not slow"
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
-      py39_macos_1:
-        ENV_FILE: ci/deps/azure-macos-39.yaml
-        CONDA_PY: "39"
-        PATTERN: "not slow"
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
-      py39_macos_2:
+
+      py39:
         ENV_FILE: ci/deps/azure-macos-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow"
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
-      py310_macos_1:
-        ENV_FILE: ci/deps/azure-macos-310.yaml
-        CONDA_PY: "310"
-        PATTERN: "not slow"
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
-      py310_macos_2:
+
+      py310:
         ENV_FILE: ci/deps/azure-macos-310.yaml
         CONDA_PY: "310"
-        PATTERN: "not slow"
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:
     - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'

ci/deps/actions-38-minimum_versions.yaml

@@ -19,7 +19,7 @@ dependencies:
   - numpy=1.18.5
   - pytz=2020.1
 
-  # optional dependencies
+  # optional dependencies, markupsafe for jinja2
   - beautifulsoup4=4.8.2
   - blosc=1.20.1
   - bottleneck=1.3.1
@@ -29,6 +29,7 @@ dependencies:
   - gcsfs=0.6.0
   - jinja2=2.11
   - lxml=4.5.0
+  - markupsafe=2.0.1
   - matplotlib=3.3.2
   - numba=0.50.1
   - numexpr=2.7.1

doc/source/user_guide/io.rst

@@ -3675,6 +3675,10 @@ should be passed to ``index_col`` and ``header``:
 
    os.remove("path_to_file.xlsx")
 
+Missing values in columns specified in ``index_col`` will be forward filled to
+allow roundtripping with ``to_excel`` for ``merged_cells=True``. To avoid forward
+filling the missing values use ``set_index`` after reading the data instead of
+``index_col``.
 
 Parsing specific columns
 ++++++++++++++++++++++++

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.4
 .. toctree::
    :maxdepth: 2
 
+   v1.4.2
    v1.4.1
    v1.4.0
 

doc/source/whatsnew/v1.4.0.rst

@@ -637,11 +637,11 @@ will continue to return :class:`Int64Index`, :class:`UInt64Index` and
 
 .. _whatsnew_140.deprecations.frame_series_append:
 
-Deprecated Frame.append and Series.append
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Deprecated DataFrame.append and Series.append
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 :meth:`DataFrame.append` and :meth:`Series.append` have been deprecated and will
-be removed in Pandas 2.0. Use :func:`pandas.concat` instead (:issue:`35407`).
+be removed in a future version. Use :func:`pandas.concat` instead (:issue:`35407`).
 
 *Deprecated syntax*
 

doc/source/whatsnew/v1.4.1.rst

@@ -53,4 +53,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.4.0..v1.4.1|HEAD
+.. contributors:: v1.4.0..v1.4.1

doc/source/whatsnew/v1.4.2.rst

@@ -0,0 +1,45 @@
+.. _whatsnew_142:
+
+What's new in 1.4.2 (March ??, 2022)
+------------------------------------
+
+These are the changes in pandas 1.4.2. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_142.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Fixed regression in :meth:`DataFrame.drop` and :meth:`Series.drop` when :class:`Index` had extension dtype and duplicates (:issue:`45860`)
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_142.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_142.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_142.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.4.1..v1.4.2|HEAD