Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/workflows/posix.yml

@@ -31,6 +31,7 @@ jobs:
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
           [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
           [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
         ]

ci/deps/actions-pypy-38.yaml

@@ -0,0 +1,20 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  # TODO: Add the rest of the dependencies in here
+  # once the other plentiful failures/segfaults
+  # with base pandas has been dealt with
+  - python=3.8
+
+  # tools
+  - cython>=0.29.24
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
+
+  # required
+  - numpy
+  - python-dateutil
+  - pytz

doc/source/whatsnew/v1.3.5.rst

@@ -22,7 +22,6 @@ Fixed regressions
 - Fixed regression in :meth:`Series.duplicated` and :meth:`Series.drop_duplicates` when Series has :class:`Categorical` dtype with boolean categories (:issue:`44351`)
 - Fixed regression in :meth:`.GroupBy.sum` with ``timedelta64[ns]`` dtype containing ``NaT`` failing to treat that value as NA (:issue:`42659`)
 - Fixed regression in :meth:`.RollingGroupby.cov` and :meth:`.RollingGroupby.corr` when ``other`` had the same shape as each group would incorrectly return superfluous groups in the result (:issue:`42915`)
-- Fixed regression where a single column ``np.matrix`` was no longer coerced to a 1d ``np.ndarray`` when added to a :class:`DataFrame` (:issue:`42376`)
 
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.4.0.rst

@@ -660,7 +660,7 @@ Conversion
 
 Strings
 ^^^^^^^
--
+- Fixed bug in checking for ``string[pyarrow]`` dtype incorrectly raising an ImportError when pyarrow is not installed (:issue:`44327`)
 -
 
 Interval
@@ -702,6 +702,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc.__setitem__` changing dtype when indexer was completely ``False`` (:issue:`37550`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` returning boolean mask instead of array of integers for a non unique and non monotonic index (:issue:`44084`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` not handling targets of ``dtype`` 'object' with NaNs correctly (:issue:`44482`)
+- Fixed regression where a single column ``np.matrix`` was no longer coerced to a 1d ``np.ndarray`` when added to a :class:`DataFrame` (:issue:`42376`)
 -
 
 Missing
@@ -710,6 +711,7 @@ Missing
 - Bug in :meth:`DataFrame.fillna` not replacing missing values when using a dict-like ``value`` and duplicate column names (:issue:`43476`)
 - Bug in constructing a :class:`DataFrame` with a dictionary ``np.datetime64`` as a value and ``dtype='timedelta64[ns]'``, or vice-versa, incorrectly casting instead of raising (:issue:`??`)
 - Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` with ``inplace=True`` not writing to the underlying array(s) in-place (:issue:`44749`)
+- Bug in :meth:`Index.fillna` incorrectly returning an un-filled :class:`Index` when NA values are present and ``downcast`` argument is specified. This now raises ``NotImplementedError`` instead; do not pass ``downcast`` argument (:issue:`44873`)
 -
 
 MultiIndex

pandas/core/dtypes/common.py

@@ -613,7 +613,7 @@ def is_dtype_equal(source, target) -> bool:
                 src = get_dtype(source)
                 if isinstance(src, ExtensionDtype):
                     return src == target
-            except (TypeError, AttributeError):
+            except (TypeError, AttributeError, ImportError):
                 return False
     elif isinstance(source, str):
         return is_dtype_equal(target, source)
@@ -622,7 +622,7 @@ def is_dtype_equal(source, target) -> bool:
         source = get_dtype(source)
         target = get_dtype(target)
         return source == target
-    except (TypeError, AttributeError):
+    except (TypeError, AttributeError, ImportError):
 
         # invalid comparison
         # object == category will hit this

pandas/core/indexes/base.py

@@ -2723,13 +2723,18 @@ def fillna(self, value=None, downcast=None):
         DataFrame.fillna : Fill NaN values of a DataFrame.
         Series.fillna : Fill NaN Values of a Series.
         """
+
         value = self._require_scalar(value)
         if self.hasnans:
             result = self.putmask(self._isnan, value)
             if downcast is None:
                 # no need to care metadata other than name
-                # because it can't have freq if
+                # because it can't have freq if it has NaTs
                 return Index._with_infer(result, name=self.name)
+            raise NotImplementedError(
+                f"{type(self).__name__}.fillna does not support 'downcast' "
+                "argument values other than 'None'."
+            )
         return self._view()
 
     def dropna(self: _IndexT, how: str_t = "any") -> _IndexT:

pandas/core/indexes/category.py

@@ -377,20 +377,6 @@ def __contains__(self, key: Any) -> bool:
 
         return contains(self, key, container=self._engine)
 
-    @doc(Index.fillna)
-    def fillna(self, value, downcast=None):
-        value = self._require_scalar(value)
-        try:
-            cat = self._data.fillna(value)
-        except (ValueError, TypeError):
-            # invalid fill_value
-            if not self.hasnans:
-                # nothing to fill, we can get away without casting
-                return self.copy()
-            return self.astype(object).fillna(value, downcast=downcast)
-
-        return type(self)._simple_new(cat, name=self.name)
-
     # TODO(2.0): remove reindex once non-unique deprecation is enforced
     def reindex(
         self, target, method=None, level=None, limit=None, tolerance=None

pandas/tests/dtypes/test_common.py

@@ -116,6 +116,7 @@ def test_period_dtype(self, dtype):
     "float": np.dtype(np.float64),
     "object": np.dtype(object),
     "category": com.pandas_dtype("category"),
+    "string": pd.StringDtype(),
 }
 
 
@@ -129,6 +130,12 @@ def test_dtype_equal(name1, dtype1, name2, dtype2):
         assert not com.is_dtype_equal(dtype1, dtype2)
 
 
+@pytest.mark.parametrize("name,dtype", list(dtypes.items()), ids=lambda x: str(x))
+def test_pyarrow_string_import_error(name, dtype):
+    # GH-44276
+    assert not com.is_dtype_equal(dtype, "string[pyarrow]")
+
+
 @pytest.mark.parametrize(
     "dtype1,dtype2",
     [

pandas/tests/indexes/categorical/test_fillna.py

@@ -25,17 +25,19 @@ def test_fillna_categorical(self):
         tm.assert_index_equal(result, expected)
 
     def test_fillna_copies_with_no_nas(self):
-        # Nothing to fill, should still get a copy
+        # Nothing to fill, should still get a copy for the Categorical method,
+        #  but OK to get a view on CategoricalIndex method
         ci = CategoricalIndex([0, 1, 1])
-        cat = ci._data
         result = ci.fillna(0)
-        assert result._values._ndarray is not cat._ndarray
-        assert result._values._ndarray.base is None
+        assert result is not ci
+        assert tm.shares_memory(result, ci)
 
-        # Same check directly on the Categorical object
+        # But at the EA level we always get a copy.
+        cat = ci._data
         result = cat.fillna(0)
         assert result._ndarray is not cat._ndarray
         assert result._ndarray.base is None
+        assert not tm.shares_memory(result, cat)
 
     def test_fillna_validates_with_no_nas(self):
         # We validate the fill value even if fillna is a no-op

pandas/tests/indexes/common.py

@@ -516,6 +516,11 @@ def test_fillna(self, index):
 
             idx = type(index)(values)
 
+            msg = "does not support 'downcast'"
+            with pytest.raises(NotImplementedError, match=msg):
+                # For now at least, we only raise if there are NAs present
+                idx.fillna(idx[0], downcast="infer")
+
             expected = np.array([False] * len(idx), dtype=bool)
             expected[1] = True
             tm.assert_numpy_array_equal(idx._isnan, expected)

pandas/tests/strings/test_strings.py

@@ -137,15 +137,11 @@ def test_repeat_mixed_object():
     tm.assert_series_equal(result, expected)
 
 
-def test_repeat_with_null(any_string_dtype):
+@pytest.mark.parametrize("arg, repeat", [[None, 4], ["b", None]])
+def test_repeat_with_null(any_string_dtype, arg, repeat):
     # GH: 31632
-    ser = Series(["a", None], dtype=any_string_dtype)
-    result = ser.str.repeat([3, 4])
-    expected = Series(["aaa", np.nan], dtype=any_string_dtype)
-    tm.assert_series_equal(result, expected)
-
-    ser = Series(["a", "b"], dtype=any_string_dtype)
-    result = ser.str.repeat([3, None])
+    ser = Series(["a", arg], dtype=any_string_dtype)
+    result = ser.str.repeat([3, repeat])
     expected = Series(["aaa", np.nan], dtype=any_string_dtype)
     tm.assert_series_equal(result, expected)
 
@@ -397,27 +393,28 @@ def test_index_not_found_raises(index_or_series, any_string_dtype):
         obj.str.index("DE")
 
 
-def test_index_wrong_type_raises(index_or_series, any_string_dtype):
+@pytest.mark.parametrize("method", ["index", "rindex"])
+def test_index_wrong_type_raises(index_or_series, any_string_dtype, method):
     obj = index_or_series([], dtype=any_string_dtype)
     msg = "expected a string object, not int"
 
     with pytest.raises(TypeError, match=msg):
-        obj.str.index(0)
-
-    with pytest.raises(TypeError, match=msg):
-        obj.str.rindex(0)
+        getattr(obj.str, method)(0)
 
 
-def test_index_missing(any_string_dtype):
+@pytest.mark.parametrize(
+    "method, exp",
+    [
+        ["index", [1, 1, 0]],
+        ["rindex", [3, 1, 2]],
+    ],
+)
+def test_index_missing(any_string_dtype, method, exp):
     ser = Series(["abcb", "ab", "bcbe", np.nan], dtype=any_string_dtype)
     expected_dtype = np.float64 if any_string_dtype == "object" else "Int64"
 
-    result = ser.str.index("b")
-    expected = Series([1, 1, 0, np.nan], dtype=expected_dtype)
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.rindex("b")
-    expected = Series([3, 1, 2, np.nan], dtype=expected_dtype)
+    result = getattr(ser.str, method)("b")
+    expected = Series(exp + [np.nan], dtype=expected_dtype)
     tm.assert_series_equal(result, expected)
 
 
@@ -488,53 +485,51 @@ def test_slice_replace(start, stop, repl, expected, any_string_dtype):
     tm.assert_series_equal(result, expected)
 
 
-def test_strip_lstrip_rstrip(any_string_dtype):
+@pytest.mark.parametrize(
+    "method, exp",
+    [
+        ["strip", ["aa", "bb", np.nan, "cc"]],
+        ["lstrip", ["aa   ", "bb \n", np.nan, "cc  "]],
+        ["rstrip", ["  aa", " bb", np.nan, "cc"]],
+    ],
+)
+def test_strip_lstrip_rstrip(any_string_dtype, method, exp):
     ser = Series(["  aa   ", " bb \n", np.nan, "cc  "], dtype=any_string_dtype)
 
-    result = ser.str.strip()
-    expected = Series(["aa", "bb", np.nan, "cc"], dtype=any_string_dtype)
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.lstrip()
-    expected = Series(["aa   ", "bb \n", np.nan, "cc  "], dtype=any_string_dtype)
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.rstrip()
-    expected = Series(["  aa", " bb", np.nan, "cc"], dtype=any_string_dtype)
+    result = getattr(ser.str, method)()
+    expected = Series(exp, dtype=any_string_dtype)
     tm.assert_series_equal(result, expected)
 
 
-def test_strip_lstrip_rstrip_mixed_object():
+@pytest.mark.parametrize(
+    "method, exp",
+    [
+        ["strip", ["aa", np.nan, "bb"]],
+        ["lstrip", ["aa  ", np.nan, "bb \t\n"]],
+        ["rstrip", ["  aa", np.nan, " bb"]],
+    ],
+)
+def test_strip_lstrip_rstrip_mixed_object(method, exp):
     ser = Series(["  aa  ", np.nan, " bb \t\n", True, datetime.today(), None, 1, 2.0])
 
-    result = ser.str.strip()
-    expected = Series(["aa", np.nan, "bb", np.nan, np.nan, np.nan, np.nan, np.nan])
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.lstrip()
-    expected = Series(
-        ["aa  ", np.nan, "bb \t\n", np.nan, np.nan, np.nan, np.nan, np.nan]
-    )
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.rstrip()
-    expected = Series(["  aa", np.nan, " bb", np.nan, np.nan, np.nan, np.nan, np.nan])
+    result = getattr(ser.str, method)()
+    expected = Series(exp + [np.nan, np.nan, np.nan, np.nan, np.nan])
     tm.assert_series_equal(result, expected)
 
 
-def test_strip_lstrip_rstrip_args(any_string_dtype):
+@pytest.mark.parametrize(
+    "method, exp",
+    [
+        ["strip", ["ABC", " BNSD", "LDFJH "]],
+        ["lstrip", ["ABCxx", " BNSD", "LDFJH xx"]],
+        ["rstrip", ["xxABC", "xx BNSD", "LDFJH "]],
+    ],
+)
+def test_strip_lstrip_rstrip_args(any_string_dtype, method, exp):
     ser = Series(["xxABCxx", "xx BNSD", "LDFJH xx"], dtype=any_string_dtype)
 
-    result = ser.str.strip("x")
-    expected = Series(["ABC", " BNSD", "LDFJH "], dtype=any_string_dtype)
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.lstrip("x")
-    expected = Series(["ABCxx", " BNSD", "LDFJH xx"], dtype=any_string_dtype)
-    tm.assert_series_equal(result, expected)
-
-    result = ser.str.rstrip("x")
-    expected = Series(["xxABC", "xx BNSD", "LDFJH "], dtype=any_string_dtype)
+    result = getattr(ser.str, method)("x")
+    expected = Series(exp, dtype=any_string_dtype)
     tm.assert_series_equal(result, expected)