Merge remote-tracking branch 'upstream/master' into unique-index

Author: TomAugspurger

ci/code_checks.sh

@@ -188,9 +188,9 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
         # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
-        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html,js} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
     else
-        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html,js} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi

doc/source/whatsnew/v1.0.0.rst

@@ -122,6 +122,7 @@ Categorical
 ^^^^^^^^^^^
 
 - Added test to assert the :func:`fillna` raises the correct ValueError message when the value isn't a value from categories (:issue:`13628`)
+- Bug in :meth:`Categorical.astype` where ``NaN`` values were handled incorrectly when casting to int (:issue:`28406`)
 -
 -
 
@@ -184,6 +185,7 @@ Indexing
 - Bug in :meth:`DataFrame.explode` would duplicate frame in the presence of duplicates in the index (:issue:`28010`)
 - Bug in reindexing a :meth:`PeriodIndex` with another type of index that contained a `Period` (:issue:`28323`) (:issue:`28337`)
 - Fix assignment of column via `.loc` with numpy non-ns datetime type (:issue:`27395`)
+- Bug in :meth:`Float64Index.astype` where ``np.inf`` was not handled properly when casting to an integer dtype (:issue:`28475`)
 
 Missing
 ^^^^^^^
@@ -240,7 +242,7 @@ Sparse
 ExtensionArray
 ^^^^^^^^^^^^^^
 
--
+- Bug in :class:`arrays.PandasArray` when setting a scalar string (:issue:`28118`, :issue:`28150`).
 -
 
 

environment.yml

@@ -36,6 +36,12 @@ dependencies:
   - nbsphinx
   - pandoc
 
+  # web (jinja2 is also needed, but it's also an optional pandas dependency)
+  - markdown
+  - feedparser
+  - pyyaml
+  - requests
+
   # testing
   - boto3
   - botocore>=1.11

pandas/_libs/testing.pyx

@@ -143,11 +143,9 @@ cpdef assert_almost_equal(a, b,
                 from pandas.util.testing import assert_attr_equal
                 assert_attr_equal('dtype', a, b, obj=obj)
 
-            try:
-                if array_equivalent(a, b, strict_nan=True):
-                    return True
-            except:
-                pass
+            if array_equivalent(a, b, strict_nan=True):
+                return True
+
         else:
             na, nb = len(a), len(b)
 

pandas/core/arrays/categorical.py

@@ -520,6 +520,9 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             if dtype == self.dtype:
                 return self
             return self._set_dtype(dtype)
+        if is_integer_dtype(dtype) and self.isna().any():
+            msg = "Cannot convert float NaN to integer"
+            raise ValueError(msg)
         return np.array(self, dtype=dtype, copy=copy)
 
     @cache_readonly

pandas/core/arrays/numpy_.py

@@ -235,15 +235,8 @@ def __setitem__(self, key, value):
         if not lib.is_scalar(value):
             value = np.asarray(value)
 
-        values = self._ndarray
-        t = np.result_type(value, values)
-        if t != self._ndarray.dtype:
-            values = values.astype(t, casting="safe")
-            values[key] = value
-            self._dtype = PandasDtype(t)
-            self._ndarray = values
-        else:
-            self._ndarray[key] = value
+        value = np.asarray(value, dtype=self._ndarray.dtype)
+        self._ndarray[key] = value
 
     def __len__(self) -> int:
         return len(self._ndarray)

pandas/core/dtypes/missing.py

@@ -445,7 +445,7 @@ def array_equivalent(left, right, strict_nan=False):
                 if not isinstance(right_value, float) or not np.isnan(right_value):
                     return False
             else:
-                if left_value != right_value:
+                if np.any(left_value != right_value):
                     return False
         return True
 

pandas/core/indexes/base.py

@@ -4735,13 +4735,13 @@ def set_value(self, arr, key, value):
     @Appender(_index_shared_docs["get_indexer_non_unique"] % _index_doc_kwargs)
     def get_indexer_non_unique(self, target):
         target = ensure_index(target)
-        if is_categorical(target):
-            target = target.astype(target.dtype.categories.dtype)
         pself, ptarget = self._maybe_promote(target)
         if pself is not self or ptarget is not target:
             return pself.get_indexer_non_unique(ptarget)
 
-        if self.is_all_dates:
+        if is_categorical(target):
+            tgt_values = np.asarray(target)
+        elif self.is_all_dates:
             tgt_values = target.asi8
         else:
             tgt_values = target._ndarray_values
@@ -4753,7 +4753,7 @@ def get_indexer_for(self, target, **kwargs):
         """
         Guaranteed return of an indexer even when non-unique.
 
-        This dispatches to get_indexer or get_indexer_nonunique
+        This dispatches to get_indexer or get_indexer_non_unique
         as appropriate.
 
         Returns

pandas/core/indexes/numeric.py

@@ -5,6 +5,7 @@
 from pandas._libs import index as libindex
 from pandas.util._decorators import Appender, cache_readonly
 
+from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.common import (
     is_bool,
     is_bool_dtype,
@@ -367,12 +368,11 @@ def astype(self, dtype, copy=True):
                 "values are required for conversion"
             ).format(dtype=dtype)
             raise TypeError(msg)
-        elif (
-            is_integer_dtype(dtype) and not is_extension_array_dtype(dtype)
-        ) and self.hasnans:
+        elif is_integer_dtype(dtype) and not is_extension_array_dtype(dtype):
             # TODO(jreback); this can change once we have an EA Index type
             # GH 13149
-            raise ValueError("Cannot convert NA to integer")
+            arr = astype_nansafe(self.values, dtype=dtype)
+            return Int64Index(arr)
         return super().astype(dtype, copy=copy)
 
     @Appender(_index_shared_docs["_convert_scalar_indexer"])

pandas/tests/arrays/test_numpy.py

@@ -211,3 +211,18 @@ def test_basic_binop():
     result = x + x
     expected = PandasArray(np.array([2, 4, 6]))
     tm.assert_extension_array_equal(result, expected)
+
+
+@pytest.mark.parametrize("dtype", [None, object])
+def test_setitem_object_typecode(dtype):
+    arr = PandasArray(np.array(["a", "b", "c"], dtype=dtype))
+    arr[0] = "t"
+    expected = PandasArray(np.array(["t", "b", "c"], dtype=dtype))
+    tm.assert_extension_array_equal(arr, expected)
+
+
+def test_setitem_no_coercion():
+    # https://github.com/pandas-dev/pandas/issues/28150
+    arr = PandasArray(np.array([1, 2, 3]))
+    with pytest.raises(ValueError, match="int"):
+        arr[0] = "a"

pandas/tests/dtypes/test_missing.py

@@ -360,6 +360,20 @@ def test_array_equivalent_str():
         )
 
 
+def test_array_equivalent_nested():
+    # reached in groupby aggregations, make sure we use np.any when checking
+    #  if the comparison is truthy
+    left = np.array([np.array([50, 70, 90]), np.array([20, 30, 40])], dtype=object)
+    right = np.array([np.array([50, 70, 90]), np.array([20, 30, 40])], dtype=object)
+
+    assert array_equivalent(left, right, strict_nan=True)
+    assert not array_equivalent(left, right[::-1], strict_nan=True)
+
+    left = np.array([np.array([50, 50, 50]), np.array([40, 40, 40])], dtype=object)
+    right = np.array([50, 40])
+    assert not array_equivalent(left, right, strict_nan=True)
+
+
 @pytest.mark.parametrize(
     "dtype, na_value",
     [

pandas/tests/extension/test_categorical.py

@@ -19,7 +19,7 @@
 import pytest
 
 import pandas as pd
-from pandas import Categorical
+from pandas import Categorical, CategoricalIndex, Timestamp
 from pandas.api.types import CategoricalDtype
 from pandas.tests.extension import base
 import pandas.util.testing as tm
@@ -197,7 +197,15 @@ def test_searchsorted(self, data_for_sorting):
 
 
 class TestCasting(base.BaseCastingTests):
-    pass
+    @pytest.mark.parametrize("cls", [Categorical, CategoricalIndex])
+    @pytest.mark.parametrize("values", [[1, np.nan], [Timestamp("2000"), pd.NaT]])
+    def test_cast_nan_to_int(self, cls, values):
+        # GH 28406
+        s = cls(values)
+
+        msg = "Cannot (cast|convert)"
+        with pytest.raises((ValueError, TypeError), match=msg):
+            s.astype(int)
 
 
 class TestArithmeticOps(base.BaseArithmeticOpsTests):

pandas/tests/indexes/interval/test_astype.py

@@ -143,7 +143,7 @@ def test_subtype_integer(self, subtype):
         tm.assert_index_equal(result, expected)
 
         # raises with NA
-        msg = "Cannot convert NA to integer"
+        msg = r"Cannot convert non-finite values \(NA or inf\) to integer"
         with pytest.raises(ValueError, match=msg):
             index.insert(0, np.nan).astype(dtype)
 

pandas/tests/indexes/test_numeric.py

@@ -242,10 +242,17 @@ def test_astype(self):
         # GH 13149
         for dtype in ["int16", "int32", "int64"]:
             i = Float64Index([0, 1.1, np.NAN])
-            msg = "Cannot convert NA to integer"
+            msg = r"Cannot convert non-finite values \(NA or inf\) to integer"
             with pytest.raises(ValueError, match=msg):
                 i.astype(dtype)
 
+    def test_cannot_cast_inf_to_int(self):
+        idx = pd.Float64Index([1, 2, np.inf])
+
+        msg = r"Cannot convert non-finite values \(NA or inf\) to integer"
+        with pytest.raises(ValueError, match=msg):
+            idx.astype(int)
+
     def test_type_coercion_fail(self, any_int_dtype):
         # see gh-15832
         msg = "Trying to coerce float values to integers"

pandas/tests/io/json/conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+
+@pytest.fixture(params=["split", "records", "index", "columns", "values"])
+def orient(request):
+    """
+    Fixture for orients excluding the table format.
+    """
+    return request.param