Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

.github/workflows/posix.yml

@@ -25,15 +25,15 @@ jobs:
     strategy:
       matrix:
         settings: [
-          [actions-38-downstream_compat.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-38-minimum_versions.yaml, "not clipboard", "", "", "", "", ""],
-          [actions-38.yaml, "not slow and not network", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-38.yaml, "not clipboard", "", "", "", "", ""],
-          [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39.yaml, "not clipboard", "", "", "", "", ""],
-          [actions-310-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-310.yaml, "not clipboard", "", "", "", "", ""],
+          [actions-38-downstream_compat.yaml, "not slow and not network", "", "", "", "", ""],
+          [actions-38-minimum_versions.yaml, "", "", "", "", "", ""],
+          [actions-38.yaml, "not slow and not network", "language-pack-it", "it_IT.utf8", "it_IT.utf8", "", ""],
+          [actions-38.yaml, "not slow and not network", "language-pack-zh-hans", "zh_CN.utf8", "zh_CN.utf8", "", ""],
+          [actions-38.yaml, "", "", "", "", "", ""],
+          [actions-pypy-38.yaml, "not slow", "", "", "", "", "--max-worker-restart 0"],
+          [actions-39.yaml, "", "", "", "", "", ""],
+          [actions-310-numpydev.yaml, "not slow and not network", "", "", "", "deprecate", "-W error"],
+          [actions-310.yaml, "", "", "", "", "", ""],
         ]
       fail-fast: false
     env:
@@ -105,7 +105,8 @@ jobs:
           hashFiles('${{ env.ENV_FILE }}') }}
 
     - name: Extra installs
-      run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 ${{ env.EXTRA_APT }}
+      # xsel for clipboard tests
+      run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }}
 
     - uses: conda-incubator/setup-miniconda@v2
       with:

asv_bench/benchmarks/frame_ctor.py

@@ -2,8 +2,10 @@
 
 import pandas as pd
 from pandas import (
+    NA,
     Categorical,
     DataFrame,
+    Float64Dtype,
     MultiIndex,
     Series,
     Timestamp,
@@ -138,6 +140,27 @@ def time_frame_from_range(self):
         self.df = DataFrame(self.data)
 
 
+class FromScalar:
+    def setup(self):
+        self.nrows = 100_000
+
+    def time_frame_from_scalar_ea_float64(self):
+        DataFrame(
+            1.0,
+            index=range(self.nrows),
+            columns=list("abc"),
+            dtype=Float64Dtype(),
+        )
+
+    def time_frame_from_scalar_ea_float64_na(self):
+        DataFrame(
+            NA,
+            index=range(self.nrows),
+            columns=list("abc"),
+            dtype=Float64Dtype(),
+        )
+
+
 class FromArrays:
 
     goal_time = 0.2

asv_bench/benchmarks/join_merge.py

@@ -216,6 +216,12 @@ def time_merge_dataframe_integer_2key(self, sort):
     def time_merge_dataframe_integer_key(self, sort):
         merge(self.df, self.df2, on="key1", sort=sort)
 
+    def time_merge_dataframe_empty_right(self, sort):
+        merge(self.left, self.right.iloc[:0], sort=sort)
+
+    def time_merge_dataframe_empty_left(self, sort):
+        merge(self.left.iloc[:0], self.right, sort=sort)
+
     def time_merge_dataframes_cross(self, sort):
         merge(self.left.loc[:2000], self.right.loc[:2000], how="cross", sort=sort)
 

azure-pipelines.yml

@@ -18,6 +18,8 @@ pr:
 variables:
   PYTEST_WORKERS: auto
   PYTEST_TARGET:  pandas
+  PATTERN: "not slow and not high_memory and not db and not network"
+  PANDAS_CI: 1
 
 jobs:
 # Mac and Linux use the same template

ci/azure/windows.yml

@@ -4,51 +4,22 @@ parameters:
 
 jobs:
 - job: ${{ parameters.name }}
+  timeoutInMinutes: 90
   pool:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py38_np18_1:
-        ENV_FILE: ci/deps/azure-windows-38.yaml
+      py38:
+        ENV_FILE: ci/deps/actions-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
 
-      py38_np18_2:
-        ENV_FILE: ci/deps/azure-windows-38.yaml
-        CONDA_PY: "38"
-        PATTERN: "not slow"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
-
-      py39_1:
-        ENV_FILE: ci/deps/azure-windows-39.yaml
-        CONDA_PY: "39"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
-
-      py39_2:
-        ENV_FILE: ci/deps/azure-windows-39.yaml
+      py39:
+        ENV_FILE: ci/deps/actions-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
-
-      py310_1:
-        ENV_FILE: ci/deps/azure-windows-310.yaml
-        CONDA_PY: "310"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
 
-      py310_2:
-        ENV_FILE: ci/deps/azure-windows-310.yaml
+      py310:
+        ENV_FILE: ci/deps/actions-310.yaml
         CONDA_PY: "310"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:
     - powershell: |
@@ -59,7 +30,7 @@ jobs:
       displayName: 'Update conda'
 
     - bash: |
-        conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
+        conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml
       displayName: 'Create anaconda environment'
     - bash: |
         source activate pandas-dev

ci/deps/azure-windows-310.yaml

@@ -24,7 +24,11 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -r fEs -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+
+if [[ "$PATTERN" ]]; then
+  PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
+fi
 
 if [[ $(uname) != "Linux"  && $(uname) != "Darwin" ]]; then
     PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/"
@@ -35,7 +39,13 @@ sh -c "$PYTEST_CMD"
 
 if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then
     # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array
-    PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS  --dist=loadfile $TEST_ARGS $COVERAGE pandas"
+    PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -n $PYTEST_WORKERS  --dist=loadfile $TEST_ARGS $COVERAGE pandas"
+
+    if [[ "$PATTERN" ]]; then
+      PYTEST_AM_CMD="$PYTEST_AM_CMD -m \"$PATTERN and arraymanager\""
+    else
+      PYTEST_AM_CMD="$PYTEST_AM_CMD -m \"arraymanager\""
+    fi
 
     echo $PYTEST_AM_CMD
     sh -c "$PYTEST_AM_CMD"

ci/deps/azure-windows-38.yaml

@@ -446,7 +446,7 @@
         "pandas-gbq": ("https://pandas-gbq.readthedocs.io/en/latest/", None),
         "py": ("https://pylib.readthedocs.io/en/latest/", None),
         "python": ("https://docs.python.org/3/", None),
-        "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None),
+        "scipy": ("https://docs.scipy.org/doc/scipy/", None),
         "statsmodels": ("https://www.statsmodels.org/devel/", None),
         "pyarrow": ("https://arrow.apache.org/docs/", None),
     }

ci/deps/azure-windows-39.yaml

@@ -16,7 +16,9 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Regression in :meth:`Series.mask` with ``inplace=True`` and ``PeriodDtype`` and an incompatible ``other`` coercing to a common dtype instead of raising (:issue:`45546`)
 - Regression in :func:`.assert_frame_equal` not respecting ``check_flags=False`` (:issue:`45554`)
+- Regression in :meth:`DataFrame.loc.__getitem__` raising ``ValueError`` when indexing on a :class:`MultiIndex` with one level (:issue:`45779`)
 - Regression in :meth:`Series.fillna` with ``downcast=False`` incorrectly downcasting ``object`` dtype (:issue:`45603`)
+- Regression in :func:`api.types.is_bool_dtype` raising an ``AttributeError`` when evaluating a categorical :class:`Series` (:issue:`45615`)
 - Regression in :meth:`DataFrame.iat` setting values leading to not propagating correctly in subsequent lookups (:issue:`45684`)
 - Regression when setting values with :meth:`DataFrame.loc` losing :class:`Index` name if :class:`DataFrame` was empty before (:issue:`45621`)
 - Regression in :meth:`~Index.join` with overlapping :class:`IntervalIndex` raising an ``InvalidIndexError`` (:issue:`45661`)
@@ -33,7 +35,6 @@ Bug fixes
 - Fixed segfault in :meth:`DataFrame.to_json` when dumping tz-aware datetimes in Python 3.10 (:issue:`42130`)
 - Stopped emitting unnecessary ``FutureWarning`` in :meth:`DataFrame.sort_values` with sparse columns (:issue:`45618`)
 - Fixed window aggregations in :meth:`DataFrame.rolling` and :meth:`Series.rolling` to skip over unused elements (:issue:`45647`)
-- Bug in :func:`api.types.is_bool_dtype` was raising an ``AttributeError`` when evaluating a categorical :class:`Series` (:issue:`45615`)
 - Fixed builtin highlighters in :class:`.Styler` to be responsive to ``NA`` with nullable dtypes (:issue:`45804`)
 
 .. ---------------------------------------------------------------------------

ci/run_tests.sh

@@ -244,6 +244,8 @@ Performance improvements
 - Performance improvement in :meth:`DataFrame.duplicated` when subset consists of only one column (:issue:`45236`)
 - Performance improvement in :meth:`.GroupBy.transform` when broadcasting values for user-defined functions (:issue:`45708`)
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions when only a single group exists (:issue:`44977`)
+- Performance improvement in :func:`merge` when left and/or right are empty (:issue:`45838`)
+- Performance improvement in :class:`DataFrame` and :class:`Series` constructors for extension dtype scalars (:issue:`45854`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -318,6 +320,7 @@ Indexing
 - Bug in getting a column from a DataFrame with an object-dtype row index with datetime-like values: the resulting Series now preserves the exact object-dtype Index from the parent DataFrame (:issue:`42950`)
 - Bug in indexing on a :class:`DatetimeIndex` with a ``np.str_`` key incorrectly raising (:issue:`45580`)
 - Bug in :meth:`CategoricalIndex.get_indexer` when index contains ``NaN`` values, resulting in elements that are in target but not present in the index to be mapped to the index of the NaN element, instead of -1 (:issue:`45361`)
+- Bug in setting large integer values into :class:`Series` with ``float32`` or ``float16`` dtype incorrectly altering these values instead of coercing to ``float64`` dtype (:issue:`45844`)
 -
 
 Missing

doc/source/_static/style/df_pipe_applydata.png

@@ -9,7 +9,7 @@ dependencies:
   - pytz
 
   # benchmarks
-  - asv
+  - asv < 0.5.0  # 2022-02-08: v0.5.0 > leads to ASV checks running > 3 hours on CI
 
   # building
   # The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms.