Merge remote-tracking branch 'upstream/master' into typ_c_parser

Author: phofl

.github/workflows/python-dev.yml

@@ -51,7 +51,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
+        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-timeout
         pip list
 
     - name: Build Pandas

.pre-commit-config.yaml

@@ -111,6 +111,8 @@ repos:
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
+            # directive should not have a space before ::
+            |\.\.\ \w+\ ::
 
             # Check for deprecated messages without sphinx directive
             |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)

azure-pipelines.yml

@@ -43,7 +43,7 @@ jobs:
         /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel setuptools && \
-        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
+        pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines pytest-timeout && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"

ci/code_checks.sh

@@ -93,8 +93,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS03, SS04, SS05, PR03, PR04, PR05, PR08, PRO9, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR08,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03
+    MSG='Validate docstrings (GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS03, SS04, SS05, PR03, PR04, PR05, PR06, PR08, PR09, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR06,PR08,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/actions-38-db-min.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # required

ci/deps/actions-38-db.yaml

@@ -8,6 +8,7 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
 

ci/deps/actions-38-locale.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - pytest-asyncio>=0.12.0
   - hypothesis>=5.5.3
 

ci/deps/actions-38-locale_slow.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-38-minimum_versions.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
   - psutil
 

ci/deps/actions-38-slow.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-38.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-39-numpydev.yaml

@@ -8,6 +8,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-39-slow.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/actions-39.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

ci/deps/azure-macos-38.yaml

@@ -8,6 +8,7 @@ dependencies:
   # tools
   - pytest>=6.0
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-azurepipelines
 

ci/deps/azure-windows-38.yaml

@@ -9,6 +9,7 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-azurepipelines
 

ci/deps/azure-windows-39.yaml

@@ -9,6 +9,7 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
   - pytest-azurepipelines
 

ci/deps/circle-38-arm64.yaml

@@ -8,6 +8,7 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
+  - pytest-timeout
   - hypothesis>=5.5.3
 
   # pandas dependencies

doc/source/user_guide/io.rst

@@ -3546,9 +3546,9 @@ with ``on_demand=True``.
 Specifying sheets
 +++++++++++++++++
 
-.. note :: The second argument is ``sheet_name``, not to be confused with ``ExcelFile.sheet_names``.
+.. note:: The second argument is ``sheet_name``, not to be confused with ``ExcelFile.sheet_names``.
 
-.. note :: An ExcelFile's attribute ``sheet_names`` provides access to a list of sheets.
+.. note:: An ExcelFile's attribute ``sheet_names`` provides access to a list of sheets.
 
 * The arguments ``sheet_name`` allows specifying the sheet or sheets to read.
 * The default value for ``sheet_name`` is 0, indicating to read the first sheet

doc/source/whatsnew/v1.3.5.rst

@@ -16,6 +16,7 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed regression in :meth:`Series.equals` when comparing floats with dtype object to None (:issue:`44190`)
 - Fixed regression in :func:`merge_asof` raising error when array was supplied as join key (:issue:`42844`)
+- Fixed regression when resampling :class:`DataFrame` with :class:`DateTimeIndex` with empty groups and ``uint8``, ``uint16`` or ``uint32`` columns incorrectly raising ``RuntimeError`` (:issue:`43329`)
 - Fixed regression in creating a :class:`DataFrame` from a timezone-aware :class:`Timestamp` scalar near a Daylight Savings Time transition (:issue:`42505`)
 - Fixed performance regression in :func:`read_csv` (:issue:`44106`)
 - Fixed regression in :meth:`Series.duplicated` and :meth:`Series.drop_duplicates` when Series has :class:`Categorical` dtype with boolean categories (:issue:`44351`)

doc/source/whatsnew/v1.4.0.rst

@@ -198,6 +198,7 @@ representation of :class:`DataFrame` objects (:issue:`4889`).
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- :meth:`concat` will preserve the ``attrs`` when it is the same for all objects and discard the ``attrs`` when they are different. (:issue:`41828`)
 - :class:`DataFrameGroupBy` operations with ``as_index=False`` now correctly retain ``ExtensionDtype`` dtypes for columns being grouped on (:issue:`41373`)
 - Add support for assigning values to ``by`` argument in :meth:`DataFrame.plot.hist` and :meth:`DataFrame.plot.box` (:issue:`15079`)
 - :meth:`Series.sample`, :meth:`DataFrame.sample`, and :meth:`.GroupBy.sample` now accept a ``np.random.Generator`` as input to ``random_state``. A generator will be more performant, especially with ``replace=False`` (:issue:`38100`)
@@ -226,6 +227,9 @@ Other enhancements
   ``USFederalHolidayCalendar``.  See also `Other API changes`_.
 - :meth:`.Rolling.var`, :meth:`.Expanding.var`, :meth:`.Rolling.std`, :meth:`.Expanding.std` now support `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`44461`)
 - :meth:`Series.info` has been added, for compatibility with :meth:`DataFrame.info` (:issue:`5167`)
+- Implemented :meth:`IntervalArray.min`, :meth:`IntervalArray.max`, as a result of which ``min`` and ``max`` now work for :class:`IntervalIndex`, :class:`Series` and :class:`DataFrame` with ``IntervalDtype`` (:issue:`44746`)
+- :meth:`UInt64Index.map` now retains ``dtype`` where possible (:issue:`44609`)
+-
 
 
 .. ---------------------------------------------------------------------------
@@ -543,7 +547,7 @@ Performance improvements
 - Performance improvement in :meth:`.GroupBy.sample`, especially when ``weights`` argument provided (:issue:`34483`)
 - Performance improvement when converting non-string arrays to string arrays (:issue:`34483`)
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions (:issue:`41598`)
-- Performance improvement in constructing :class:`DataFrame` objects (:issue:`42631`, :issue:`43142`, :issue:`43147`, :issue:`43307`, :issue:`43144`)
+- Performance improvement in constructing :class:`DataFrame` objects (:issue:`42631`, :issue:`43142`, :issue:`43147`, :issue:`43307`, :issue:`43144`, :issue:`44826`)
 - Performance improvement in :meth:`GroupBy.shift` when ``fill_value`` argument is provided (:issue:`26615`)
 - Performance improvement in :meth:`DataFrame.corr` for ``method=pearson`` on data without missing values (:issue:`40956`)
 - Performance improvement in some :meth:`GroupBy.apply` operations (:issue:`42992`, :issue:`43578`)
@@ -640,6 +644,7 @@ Numeric
 - Bug in arithmetic operations involving :class:`RangeIndex` where the result would have the incorrect ``name`` (:issue:`43962`)
 - Bug in arithmetic operations involving :class:`Series` where the result could have the incorrect ``name`` when the operands having matching NA or matching tuple names (:issue:`44459`)
 - Bug in division with ``IntegerDtype`` or ``BooleanDtype`` array and NA scalar incorrectly raising (:issue:`44685`)
+- Bug in multiplying a :class:`Series` with ``FloatingDtype`` with a timedelta-like scalar incorrectly raising (:issue:`44772`)
 -
 
 Conversion
@@ -649,7 +654,7 @@ Conversion
 - Bug in :class:`IntegerDtype` not allowing coercion from string dtype (:issue:`25472`)
 - Bug in :func:`to_datetime` with ``arg:xr.DataArray`` and ``unit="ns"`` specified raises TypeError (:issue:`44053`)
 - Bug in :meth:`DataFrame.convert_dtypes` not returning the correct type when a subclass does not overload :meth:`_constructor_sliced` (:issue:`43201`)
--
+- Bug in :meth:`DataFrame.astype` not propagating ``attrs`` from the original :class:`DataFrame` (:issue:`44414`)
 
 Strings
 ^^^^^^^
@@ -701,6 +706,7 @@ Missing
 - Bug in :meth:`DataFrame.fillna` with limit and no method ignores axis='columns' or ``axis = 1`` (:issue:`40989`)
 - Bug in :meth:`DataFrame.fillna` not replacing missing values when using a dict-like ``value`` and duplicate column names (:issue:`43476`)
 - Bug in constructing a :class:`DataFrame` with a dictionary ``np.datetime64`` as a value and ``dtype='timedelta64[ns]'``, or vice-versa, incorrectly casting instead of raising (:issue:`??`)
+- Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` with ``inplace=True`` not writing to the underlying array(s) in-place (:issue:`44749`)
 -
 
 MultiIndex
@@ -742,6 +748,8 @@ I/O
 - Bug in :func:`read_csv` raising ``AttributeError`` when attempting to read a .csv file and infer index column dtype from an nullable integer type (:issue:`44079`)
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` with ``compression`` set to ``'zip'`` no longer create a zip file containing a file ending with ".zip". Instead, they try to infer the inner file name more smartly. (:issue:`39465`)
 - Bug in :func:`read_csv` when passing simultaneously a parser in ``date_parser`` and ``parse_dates=False``, the parsing was still called (:issue:`44366`)
+- Bug in :func:`read_csv` silently ignoring errors when failling to create a memory-mapped file (:issue:`44766`)
+-
 
 Period
 ^^^^^^

environment.yml

@@ -61,6 +61,7 @@ dependencies:
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
+  - pytest-timeout
   - pytest-asyncio
   - pytest-instafail
 

pandas/_config/config.py

@@ -642,7 +642,6 @@ def _warn_if_deprecated(key: str) -> bool:
     d = _get_deprecated_option(key)
     if d:
         if d.msg:
-            print(d.msg)
             warnings.warn(d.msg, FutureWarning)
         else:
             msg = f"'{key}' is deprecated"

pandas/_testing/__init__.py

@@ -110,12 +110,15 @@
     UInt64Index,
 )
 from pandas.core.arrays import (
+    BaseMaskedArray,
     DatetimeArray,
+    ExtensionArray,
     PandasArray,
     PeriodArray,
     TimedeltaArray,
     period_array,
 )
+from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
 
 if TYPE_CHECKING:
     from pandas import (
@@ -1050,3 +1053,53 @@ def at(x):
 
 def iat(x):
     return x.iat
+
+
+# -----------------------------------------------------------------------------
+
+
+def shares_memory(left, right) -> bool:
+    """
+    Pandas-compat for np.shares_memory.
+    """
+    if isinstance(left, np.ndarray) and isinstance(right, np.ndarray):
+        return np.shares_memory(left, right)
+    elif isinstance(left, np.ndarray):
+        # Call with reversed args to get to unpacking logic below.
+        return shares_memory(right, left)
+
+    if isinstance(left, RangeIndex):
+        return False
+    if isinstance(left, MultiIndex):
+        return shares_memory(left._codes, right)
+    if isinstance(left, (Index, Series)):
+        return shares_memory(left._values, right)
+
+    if isinstance(left, NDArrayBackedExtensionArray):
+        return shares_memory(left._ndarray, right)
+    if isinstance(left, pd.core.arrays.SparseArray):
+        return shares_memory(left.sp_values, right)
+
+    if isinstance(left, ExtensionArray) and left.dtype == "string[pyarrow]":
+        # https://github.com/pandas-dev/pandas/pull/43930#discussion_r736862669
+        if isinstance(right, ExtensionArray) and right.dtype == "string[pyarrow]":
+            # error: "ExtensionArray" has no attribute "_data"
+            left_pa_data = left._data  # type: ignore[attr-defined]
+            # error: "ExtensionArray" has no attribute "_data"
+            right_pa_data = right._data  # type: ignore[attr-defined]
+            left_buf1 = left_pa_data.chunk(0).buffers()[1]
+            right_buf1 = right_pa_data.chunk(0).buffers()[1]
+            return left_buf1 == right_buf1
+
+    if isinstance(left, BaseMaskedArray) and isinstance(right, BaseMaskedArray):
+        # By convention, we'll say these share memory if they share *either*
+        #  the _data or the _mask
+        return np.shares_memory(left._data, right._data) or np.shares_memory(
+            left._mask, right._mask
+        )
+
+    if isinstance(left, DataFrame) and len(left._mgr.arrays) == 1:
+        arr = left._mgr.arrays[0]
+        return shares_memory(arr, right)
+
+    raise NotImplementedError(type(left), type(right))

pandas/_testing/_warnings.py

@@ -147,12 +147,16 @@ def _assert_caught_no_extra_warnings(
 
     for actual_warning in caught_warnings:
         if _is_unexpected_warning(actual_warning, expected_warning):
-            unclosed = "unclosed transport <asyncio.sslproto._SSLProtocolTransport"
-            if actual_warning.category == ResourceWarning and unclosed in str(
-                actual_warning.message
+            # GH 44732: Don't make the CI flaky by filtering SSL-related
+            # ResourceWarning from dependencies
+            # GH#38630 pytest.filterwarnings does not suppress these.
+            unclosed_ssl = (
+                "unclosed transport <asyncio.sslproto._SSLProtocolTransport",
+                "unclosed <ssl.SSLSocket",
+            )
+            if actual_warning.category == ResourceWarning and any(
+                msg in str(actual_warning.message) for msg in unclosed_ssl
             ):
-                # FIXME(GH#38630): kludge because pytest.filterwarnings does not
-                #  suppress these
                 continue
 
             extra_warnings.append(

pandas/_testing/asserters.py

@@ -1385,7 +1385,8 @@ def assert_equal(left, right, **kwargs):
         assert kwargs == {}
         assert left == right
     else:
-        raise NotImplementedError(type(left))
+        assert kwargs == {}
+        assert_almost_equal(left, right)
 
 
 def assert_sp_array_equal(left, right):

pandas/core/algorithms.py

@@ -633,7 +633,7 @@ def factorize(
         is a Categorical. When `values` is some other pandas object, an
         `Index` is returned. Otherwise, a 1-D ndarray is returned.
 
-        .. note ::
+        .. note::
 
            Even if there's a missing value in `values`, `uniques` will
            *not* contain an entry for it.

pandas/core/arraylike.py

@@ -326,13 +326,16 @@ def array_ufunc(self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any)
         reconstruct_kwargs = {}
 
     def reconstruct(result):
+        if ufunc.nout > 1:
+            # np.modf, np.frexp, np.divmod
+            return tuple(_reconstruct(x) for x in result)
+
+        return _reconstruct(result)
+
+    def _reconstruct(result):
         if lib.is_scalar(result):
             return result
 
-        if isinstance(result, tuple):
-            # np.modf, np.frexp, np.divmod
-            return tuple(reconstruct(x) for x in result)
-
         if result.ndim != self.ndim:
             if method == "outer":
                 if self.ndim == 2:
@@ -367,10 +370,12 @@ def reconstruct(result):
         return result
 
     if "out" in kwargs:
+        # e.g. test_multiindex_get_loc
         result = dispatch_ufunc_with_out(self, ufunc, method, *inputs, **kwargs)
         return reconstruct(result)
 
     if method == "reduce":
+        # e.g. test.series.test_ufunc.test_reduce
         result = dispatch_reduction_ufunc(self, ufunc, method, *inputs, **kwargs)
         if result is not NotImplemented:
             return result