Merge remote-tracking branch 'upstream/master' into typ_c_parser

Author: phofl

.github/workflows/posix.yml

@@ -31,6 +31,7 @@ jobs:
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
           [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
           [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
         ]

.github/workflows/python-dev.yml

@@ -49,10 +49,9 @@ jobs:
       shell: bash
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        # TODO: unpin
-        pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple "numpy==1.23.0.dev0+101.ga81535a36"
+        pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-timeout
+        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas

azure-pipelines.yml

@@ -43,7 +43,7 @@ jobs:
         /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel setuptools && \
-        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines pytest-timeout && \
+        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"

ci/deps/actions-38-db-min.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # required

ci/deps/actions-38-db.yaml

@@ -8,7 +8,6 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
 

ci/deps/actions-38-locale.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - pytest-asyncio>=0.12.0
   - hypothesis>=5.5.3
 

ci/deps/actions-38-locale_slow.yaml

@@ -10,7 +10,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-38-minimum_versions.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
   - psutil
 

ci/deps/actions-38-slow.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-38.yaml

@@ -10,7 +10,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-39-numpydev.yaml

@@ -8,7 +8,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies
@@ -19,6 +18,5 @@ dependencies:
     - cython==0.29.24 # GH#34014
     - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
-    # TODO: Unpin
-    - "numpy==1.23.0.dev0+101.ga81535a36"
+    - "numpy"
     - "scipy"

ci/deps/actions-39-slow.yaml

@@ -10,7 +10,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-39.yaml

@@ -9,7 +9,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-pypy-38.yaml

@@ -0,0 +1,20 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  # TODO: Add the rest of the dependencies in here
+  # once the other plentiful failures/segfaults
+  # with base pandas has been dealt with
+  - python=3.8
+
+  # tools
+  - cython>=0.29.24
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
+
+  # required
+  - numpy
+  - python-dateutil
+  - pytz

ci/deps/azure-macos-38.yaml

@@ -8,7 +8,6 @@ dependencies:
   # tools
   - pytest>=6.0
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-azurepipelines
 

ci/deps/azure-windows-38.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-azurepipelines
 

ci/deps/azure-windows-39.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-azurepipelines
 

ci/deps/circle-38-arm64.yaml

@@ -8,7 +8,6 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
-  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

doc/source/whatsnew/v1.3.5.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_135:
 
-What's new in 1.3.5 (November ??, 2021)
+What's new in 1.3.5 (December 12, 2021)
 ---------------------------------------
 
 These are the changes in pandas 1.3.5. See :ref:`release` for a full changelog
@@ -21,25 +21,8 @@ Fixed regressions
 - Fixed performance regression in :func:`read_csv` (:issue:`44106`)
 - Fixed regression in :meth:`Series.duplicated` and :meth:`Series.drop_duplicates` when Series has :class:`Categorical` dtype with boolean categories (:issue:`44351`)
 - Fixed regression in :meth:`.GroupBy.sum` with ``timedelta64[ns]`` dtype containing ``NaT`` failing to treat that value as NA (:issue:`42659`)
--
+- Fixed regression in :meth:`.RollingGroupby.cov` and :meth:`.RollingGroupby.corr` when ``other`` had the same shape as each group would incorrectly return superfluous groups in the result (:issue:`42915`)
 
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_135.bug_fixes:
-
-Bug fixes
-~~~~~~~~~
--
--
-
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_135.other:
-
-Other
-~~~~~
--
--
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.4.0.rst

@@ -219,7 +219,7 @@ Other enhancements
 - :meth:`DataFrame.dropna` now accepts a single label as ``subset`` along with array-like (:issue:`41021`)
 - :class:`ExcelWriter` argument ``if_sheet_exists="overlay"`` option added (:issue:`40231`)
 - :meth:`read_excel` now accepts a ``decimal`` argument that allow the user to specify the decimal point when parsing string columns to numeric (:issue:`14403`)
-- :meth:`.GroupBy.mean` now supports `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`43731`)
+- :meth:`.GroupBy.mean`, :meth:`.GroupBy.std`, and :meth:`.GroupBy.var` now supports `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`43731`, :issue:`44862`)
 - :meth:`Timestamp.isoformat`, now handles the ``timespec`` argument from the base :class:``datetime`` class (:issue:`26131`)
 - :meth:`NaT.to_numpy` ``dtype`` argument is now respected, so ``np.timedelta64`` can be returned (:issue:`44460`)
 - New option ``display.max_dir_items`` customizes the number of columns added to :meth:`Dataframe.__dir__` and suggested for tab completion (:issue:`37996`)
@@ -229,6 +229,8 @@ Other enhancements
 - :meth:`Series.info` has been added, for compatibility with :meth:`DataFrame.info` (:issue:`5167`)
 - Implemented :meth:`IntervalArray.min`, :meth:`IntervalArray.max`, as a result of which ``min`` and ``max`` now work for :class:`IntervalIndex`, :class:`Series` and :class:`DataFrame` with ``IntervalDtype`` (:issue:`44746`)
 - :meth:`UInt64Index.map` now retains ``dtype`` where possible (:issue:`44609`)
+- :meth:`read_json` can now parse unsigned long long integers (:issue:`26068`)
+- :meth:`DataFrame.take` now raises a ``TypeError`` when passed a scalar for the indexer (:issue:`42875`)
 -
 
 
@@ -658,7 +660,7 @@ Conversion
 
 Strings
 ^^^^^^^
--
+- Fixed bug in checking for ``string[pyarrow]`` dtype incorrectly raising an ImportError when pyarrow is not installed (:issue:`44327`)
 -
 
 Interval
@@ -697,8 +699,10 @@ Indexing
 - Bug in :meth:`DataFrame.loc.__getitem__` incorrectly raising ``KeyError`` when selecting a single column with a boolean key (:issue:`44322`).
 - Bug in setting :meth:`DataFrame.iloc` with a single ``ExtensionDtype`` column and setting 2D values e.g. ``df.iloc[:] = df.values`` incorrectly raising (:issue:`44514`)
 - Bug in indexing on columns with ``loc`` or ``iloc`` using a slice with a negative step with ``ExtensionDtype`` columns incorrectly raising (:issue:`44551`)
+- Bug in :meth:`DataFrame.loc.__setitem__` changing dtype when indexer was completely ``False`` (:issue:`37550`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` returning boolean mask instead of array of integers for a non unique and non monotonic index (:issue:`44084`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` not handling targets of ``dtype`` 'object' with NaNs correctly (:issue:`44482`)
+- Fixed regression where a single column ``np.matrix`` was no longer coerced to a 1d ``np.ndarray`` when added to a :class:`DataFrame` (:issue:`42376`)
 -
 
 Missing
@@ -707,13 +711,15 @@ Missing
 - Bug in :meth:`DataFrame.fillna` not replacing missing values when using a dict-like ``value`` and duplicate column names (:issue:`43476`)
 - Bug in constructing a :class:`DataFrame` with a dictionary ``np.datetime64`` as a value and ``dtype='timedelta64[ns]'``, or vice-versa, incorrectly casting instead of raising (:issue:`??`)
 - Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` with ``inplace=True`` not writing to the underlying array(s) in-place (:issue:`44749`)
+- Bug in :meth:`Index.fillna` incorrectly returning an un-filled :class:`Index` when NA values are present and ``downcast`` argument is specified. This now raises ``NotImplementedError`` instead; do not pass ``downcast`` argument (:issue:`44873`)
 -
 
 MultiIndex
 ^^^^^^^^^^
 - Bug in :meth:`MultiIndex.get_loc` where the first level is a :class:`DatetimeIndex` and a string key is passed (:issue:`42465`)
 - Bug in :meth:`MultiIndex.reindex` when passing a ``level`` that corresponds to an ``ExtensionDtype`` level (:issue:`42043`)
 - Bug in :meth:`MultiIndex.get_loc` raising ``TypeError`` instead of ``KeyError`` on nested tuple (:issue:`42440`)
+- Bug in :meth:`MultiIndex.union` setting wrong ``sortorder`` causing errors in subsequent indexing operations with slices (:issue:`44752`)
 - Bug in :meth:`MultiIndex.putmask` where the other value was also a :class:`MultiIndex` (:issue:`43212`)
 -
 
@@ -749,6 +755,7 @@ I/O
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` with ``compression`` set to ``'zip'`` no longer create a zip file containing a file ending with ".zip". Instead, they try to infer the inner file name more smartly. (:issue:`39465`)
 - Bug in :func:`read_csv` when passing simultaneously a parser in ``date_parser`` and ``parse_dates=False``, the parsing was still called (:issue:`44366`)
 - Bug in :func:`read_csv` silently ignoring errors when failling to create a memory-mapped file (:issue:`44766`)
+- Bug in :func:`read_csv` when passing a ``tempfile.SpooledTemporaryFile`` opened in binary mode (:issue:`44748`)
 -
 
 Period

environment.yml

@@ -61,7 +61,6 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-timeout
   - pytest-asyncio
   - pytest-instafail
 

pandas/_libs/interval.pyx

@@ -516,9 +516,9 @@ def intervals_to_interval_bounds(ndarray intervals, bint validate_closed=True):
 
     Returns
     -------
-    tuple of tuples
-        left : (ndarray, object, array)
-        right : (ndarray, object, array)
+    tuple of
+        left : ndarray
+        right : ndarray
         closed: str
     """
     cdef:

pandas/_libs/join.pyi

@@ -55,39 +55,39 @@ def asof_join_backward_on_X_by_Y(
     left_by_values: np.ndarray,  # by_t[:]
     right_by_values: np.ndarray,  # by_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_forward_on_X_by_Y(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     left_by_values: np.ndarray,  # by_t[:]
     right_by_values: np.ndarray,  # by_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_nearest_on_X_by_Y(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     left_by_values: np.ndarray,  # by_t[:]
     right_by_values: np.ndarray,  # by_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_backward(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_forward(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_nearest(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...

pandas/_libs/missing.pyi

@@ -0,0 +1,15 @@
+import numpy as np
+from numpy import typing as npt
+
+class NAType: ...
+
+NA: NAType
+
+def is_matching_na(
+    left: object, right: object, nan_matches_none: bool = ...
+) -> bool: ...
+def isposinf_scalar(val: object) -> bool: ...
+def isneginf_scalar(val: object) -> bool: ...
+def checknull(val: object, inf_as_na: bool = ...) -> bool: ...
+def isnaobj(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
+def is_numeric_na(values: np.ndarray) -> npt.NDArray[np.bool_]: ...

pandas/_libs/ops.pyi

@@ -1,6 +1,7 @@
 from typing import (
     Any,
     Callable,
+    Iterable,
     Literal,
     overload,
 )
@@ -35,15 +36,15 @@ def vec_binop(
 @overload
 def maybe_convert_bool(
     arr: npt.NDArray[np.object_],
-    true_values=...,
-    false_values=...,
+    true_values: Iterable = ...,
+    false_values: Iterable = ...,
     convert_to_masked_nullable: Literal[False] = ...,
 ) -> tuple[np.ndarray, None]: ...
 @overload
 def maybe_convert_bool(
     arr: npt.NDArray[np.object_],
-    true_values=...,
-    false_values=...,
+    true_values: Iterable = ...,
+    false_values: Iterable = ...,
     *,
     convert_to_masked_nullable: Literal[True],
 ) -> tuple[np.ndarray, np.ndarray]: ...

pandas/_libs/ops_dispatch.pyx

@@ -34,7 +34,7 @@ UFUNC_ALIASES = {
     "true_divide": "truediv",
     "power": "pow",
     "remainder": "mod",
-    "divide": "div",
+    "divide": "truediv",
     "equal": "eq",
     "not_equal": "ne",
     "less": "lt",

pandas/_libs/src/ujson/lib/ultrajson.h

@@ -297,6 +297,7 @@ typedef struct __JSONObjectDecoder {
   JSOBJ (*endArray)(void *prv, JSOBJ obj);
   JSOBJ (*newInt)(void *prv, JSINT32 value);
   JSOBJ (*newLong)(void *prv, JSINT64 value);
+  JSOBJ (*newUnsignedLong)(void *prv, JSUINT64 value);
   JSOBJ (*newDouble)(void *prv, double value);
   void (*releaseObject)(void *prv, JSOBJ obj, void *decoder);
   JSPFN_MALLOC malloc;