Merge pull request #60 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

doc/source/whatsnew/v1.1.0.rst

@@ -103,7 +103,7 @@ Bug fixes
 
 Categorical
 ^^^^^^^^^^^
-
+- Bug when passing categorical data to :class:`Index` constructor along with ``dtype=object`` incorrectly returning a :class:`CategoricalIndex` instead of object-dtype :class:`Index` (:issue:`32167`)
 -
 -
 

pandas/conftest.py

@@ -986,7 +986,7 @@ def _gen_mi():
     "uint": tm.makeUIntIndex(100),
     "range": tm.makeRangeIndex(100),
     "float": tm.makeFloatIndex(100),
-    "bool": tm.makeBoolIndex(2),
+    "bool": tm.makeBoolIndex(10),
     "categorical": tm.makeCategoricalIndex(100),
     "interval": tm.makeIntervalIndex(100),
     "empty": Index([]),
@@ -998,6 +998,15 @@ def _gen_mi():
 
 @pytest.fixture(params=indices_dict.keys())
 def indices(request):
+    """
+    Fixture for many "simple" kinds of indices.
+
+    These indices are unlikely to cover corner cases, e.g.
+        - no names
+        - no NaTs/NaNs
+        - no values near implementation bounds
+        - ...
+    """
     # copy to avoid mutation, e.g. setting .name
     return indices_dict[request.param].copy()
 
@@ -1015,6 +1024,14 @@ def _create_series(index):
 }
 
 
+@pytest.fixture
+def series_with_simple_index(indices):
+    """
+    Fixture for tests on series with changing types of indices.
+    """
+    return _create_series(indices)
+
+
 _narrow_dtypes = [
     np.float16,
     np.float32,

pandas/core/indexes/base.py

@@ -304,7 +304,7 @@ def __new__(
             # Delay import for perf. https://github.com/pandas-dev/pandas/pull/31423
             from pandas.core.indexes.category import CategoricalIndex
 
-            return CategoricalIndex(data, dtype=dtype, copy=copy, name=name, **kwargs)
+            return _maybe_asobject(dtype, CategoricalIndex, data, copy, name, **kwargs)
 
         # interval
         elif is_interval_dtype(data) or is_interval_dtype(dtype):
@@ -3146,8 +3146,7 @@ def _convert_slice_indexer(self, key: slice, kind: str_t):
         def is_int(v):
             return v is None or is_integer(v)
 
-        is_null_slicer = start is None and stop is None
-        is_index_slice = is_int(start) and is_int(stop)
+        is_index_slice = is_int(start) and is_int(stop) and is_int(step)
         is_positional = is_index_slice and not (
             self.is_integer() or self.is_categorical()
         )
@@ -3177,7 +3176,7 @@ def is_int(v):
             except KeyError:
                 pass
 
-        if is_null_slicer:
+        if com.is_null_slice(key):
             indexer = key
         elif is_positional:
             indexer = key

pandas/core/indexing.py

@@ -732,14 +732,15 @@ def _getitem_lowerdim(self, tup: Tuple):
             raise IndexingError("Too many indexers. handle elsewhere")
 
         for i, key in enumerate(tup):
-            if is_label_like(key) or isinstance(key, tuple):
+            if is_label_like(key):
+                # We don't need to check for tuples here because those are
+                #  caught by the _is_nested_tuple_indexer check above.
                 section = self._getitem_axis(key, axis=i)
 
-                # we have yielded a scalar ?
-                if not is_list_like_indexer(section):
-                    return section
-
-                elif section.ndim == self.ndim:
+                # We should never have a scalar section here, because
+                #  _getitem_lowerdim is only called after a check for
+                #  is_scalar_access, which that would be.
+                if section.ndim == self.ndim:
                     # we're in the middle of slicing through a MultiIndex
                     # revise the key wrt to `section` by inserting an _NS
                     new_key = tup[:i] + (_NS,) + tup[i + 1 :]
@@ -757,7 +758,7 @@ def _getitem_lowerdim(self, tup: Tuple):
                 # slice returns a new object.
                 if com.is_null_slice(new_key):
                     return section
-                # This is an elided recursive call to iloc/loc/etc'
+                # This is an elided recursive call to iloc/loc
                 return getattr(section, self.name)[new_key]
 
         raise IndexingError("not applicable")
@@ -1013,15 +1014,7 @@ def _getitem_tuple(self, tup: Tuple):
         return self._getitem_tuple_same_dim(tup)
 
     def _get_label(self, label, axis: int):
-        if self.ndim == 1:
-            # for perf reasons we want to try _xs first
-            # as its basically direct indexing
-            # but will fail when the index is not present
-            # see GH5667
-            return self.obj._xs(label, axis=axis)
-        elif isinstance(label, tuple) and isinstance(label[axis], slice):
-            raise IndexingError("no slices here, handle elsewhere")
-
+        # GH#5667 this will fail if the label is not present in the axis.
         return self.obj._xs(label, axis=axis)
 
     def _handle_lowerdim_multi_index_axis0(self, tup: Tuple):
@@ -1298,7 +1291,7 @@ def _validate_read_indexer(
 
             # We (temporarily) allow for some missing keys with .loc, except in
             # some cases (e.g. setting) in which "raise_missing" will be False
-            if not (self.name == "loc" and not raise_missing):
+            if raise_missing:
                 not_found = list(set(key) - set(ax))
                 raise KeyError(f"{not_found} not in index")
 
@@ -1363,10 +1356,7 @@ def _validate_key(self, key, axis: int):
         else:
             raise ValueError(f"Can only index by location with a [{self._valid_types}]")
 
-    def _has_valid_setitem_indexer(self, indexer):
-        self._has_valid_positional_setitem_indexer(indexer)
-
-    def _has_valid_positional_setitem_indexer(self, indexer) -> bool:
+    def _has_valid_setitem_indexer(self, indexer) -> bool:
         """
         Validate that a positional indexer cannot enlarge its target
         will raise if needed, does not modify the indexer externally.
@@ -1376,7 +1366,7 @@ def _has_valid_positional_setitem_indexer(self, indexer) -> bool:
         bool
         """
         if isinstance(indexer, dict):
-            raise IndexError(f"{self.name} cannot enlarge its target object")
+            raise IndexError("iloc cannot enlarge its target object")
         else:
             if not isinstance(indexer, tuple):
                 indexer = _tuplify(self.ndim, indexer)
@@ -1389,11 +1379,9 @@ def _has_valid_positional_setitem_indexer(self, indexer) -> bool:
                     pass
                 elif is_integer(i):
                     if i >= len(ax):
-                        raise IndexError(
-                            f"{self.name} cannot enlarge its target object"
-                        )
+                        raise IndexError("iloc cannot enlarge its target object")
                 elif isinstance(i, dict):
-                    raise IndexError(f"{self.name} cannot enlarge its target object")
+                    raise IndexError("iloc cannot enlarge its target object")
 
         return True
 
@@ -1520,8 +1508,8 @@ def _convert_to_indexer(self, key, axis: int, is_setter: bool = False):
             return key
 
         elif is_float(key):
+            # _validate_indexer call will always raise
             labels._validate_indexer("positional", key, "iloc")
-            return key
 
         self._validate_key(key, axis)
         return key
@@ -1582,7 +1570,7 @@ def _setitem_with_indexer(self, indexer, value):
                     # this correctly sets the dtype and avoids cache issues
                     # essentially this separates out the block that is needed
                     # to possibly be modified
-                    if self.ndim > 1 and i == self.obj._info_axis_number:
+                    if self.ndim > 1 and i == info_axis:
 
                         # add the new item, and set the value
                         # must have all defined axes if we have a scalar

pandas/core/internals/managers.py

@@ -8,6 +8,7 @@
 import numpy as np
 
 from pandas._libs import Timedelta, Timestamp, internals as libinternals, lib
+from pandas._typing import DtypeObj
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core.dtypes.cast import (
@@ -847,7 +848,7 @@ def to_dict(self, copy: bool = True):
 
         return {dtype: self.combine(blocks, copy=copy) for dtype, blocks in bd.items()}
 
-    def fast_xs(self, loc):
+    def fast_xs(self, loc: int):
         """
         get a cross sectional for a given location in the
         items ; handle dups
@@ -883,12 +884,12 @@ def fast_xs(self, loc):
             for i, rl in enumerate(blk.mgr_locs):
                 result[rl] = blk.iget((i, loc))
 
-        if is_extension_array_dtype(dtype):
+        if isinstance(dtype, ExtensionDtype):
             result = dtype.construct_array_type()._from_sequence(result, dtype=dtype)
 
         return result
 
-    def consolidate(self):
+    def consolidate(self) -> "BlockManager":
         """
         Join together blocks having same dtype
 
@@ -940,7 +941,7 @@ def get(self, item):
                 new_axis=self.items[indexer], indexer=indexer, axis=0, allow_dups=True
             )
 
-    def iget(self, i):
+    def iget(self, i: int) -> "SingleBlockManager":
         """
         Return the data as a SingleBlockManager.
         """
@@ -1377,7 +1378,7 @@ def canonicalize(block):
             block.equals(oblock) for block, oblock in zip(self_blocks, other_blocks)
         )
 
-    def unstack(self, unstacker_func, fill_value):
+    def unstack(self, unstacker_func, fill_value) -> "BlockManager":
         """
         Return a BlockManager with all blocks unstacked..
 
@@ -1396,8 +1397,8 @@ def unstack(self, unstacker_func, fill_value):
         dummy = unstacker_func(np.empty((0, 0)), value_columns=self.items)
         new_columns = dummy.get_new_columns()
         new_index = dummy.get_new_index()
-        new_blocks = []
-        columns_mask = []
+        new_blocks: List[Block] = []
+        columns_mask: List[np.ndarray] = []
 
         for blk in self.blocks:
             blocks, mask = blk._unstack(
@@ -1478,7 +1479,7 @@ def _post_setstate(self):
         pass
 
     @property
-    def _block(self):
+    def _block(self) -> Block:
         return self.blocks[0]
 
     @property
@@ -1495,14 +1496,14 @@ def _blklocs(self):
         """ compat with BlockManager """
         return None
 
-    def get_slice(self, slobj, axis=0):
+    def get_slice(self, slobj: slice, axis: int = 0) -> "SingleBlockManager":
         if axis >= self.ndim:
             raise IndexError("Requested axis not found in manager")
 
-        return type(self)(self._block._slice(slobj), self.index[slobj], fastpath=True,)
+        return type(self)(self._block._slice(slobj), self.index[slobj], fastpath=True)
 
     @property
-    def index(self):
+    def index(self) -> Index:
         return self.axes[0]
 
     @property
@@ -1516,7 +1517,7 @@ def array_dtype(self):
     def get_dtype_counts(self):
         return {self.dtype.name: 1}
 
-    def get_dtypes(self):
+    def get_dtypes(self) -> np.ndarray:
         return np.array([self._block.dtype])
 
     def external_values(self):
@@ -1527,15 +1528,15 @@ def internal_values(self):
         """The array that Series._values returns"""
         return self._block.internal_values()
 
-    def get_values(self):
+    def get_values(self) -> np.ndarray:
         """ return a dense type view """
         return np.array(self._block.to_dense(), copy=False)
 
     @property
     def _can_hold_na(self) -> bool:
         return self._block._can_hold_na
 
-    def is_consolidated(self):
+    def is_consolidated(self) -> bool:
         return True
 
     def _consolidate_check(self):
@@ -1813,9 +1814,7 @@ def _shape_compat(x):
     return stacked, placement
 
 
-def _interleaved_dtype(
-    blocks: List[Block],
-) -> Optional[Union[np.dtype, ExtensionDtype]]:
+def _interleaved_dtype(blocks: Sequence[Block]) -> Optional[DtypeObj]:
     """
     Find the common dtype for `blocks`.
 
@@ -1825,7 +1824,7 @@ def _interleaved_dtype(
 
     Returns
     -------
-    dtype : Optional[Union[np.dtype, ExtensionDtype]]
+    dtype : np.dtype, ExtensionDtype, or None
         None is returned when `blocks` is empty.
     """
     if not len(blocks):

pandas/core/ops/__init__.py

@@ -5,22 +5,17 @@
 """
 import datetime
 import operator
-from typing import TYPE_CHECKING, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Optional, Set, Tuple
 
 import numpy as np
 
 from pandas._libs import Timedelta, Timestamp, lib
 from pandas._libs.ops_dispatch import maybe_dispatch_ufunc_to_dunder_op  # noqa:F401
-from pandas._typing import Level
+from pandas._typing import ArrayLike, Level
 from pandas.util._decorators import Appender
 
 from pandas.core.dtypes.common import is_list_like, is_timedelta64_dtype
-from pandas.core.dtypes.generic import (
-    ABCDataFrame,
-    ABCExtensionArray,
-    ABCIndexClass,
-    ABCSeries,
-)
+from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core.construction import extract_array
@@ -451,10 +446,7 @@ def _align_method_SERIES(left, right, align_asobject=False):
 
 
 def _construct_result(
-    left: ABCSeries,
-    result: Union[np.ndarray, ABCExtensionArray],
-    index: ABCIndexClass,
-    name,
+    left: ABCSeries, result: ArrayLike, index: ABCIndexClass, name,
 ):
     """
     Construct an appropriately-labelled Series from the result of an op.
@@ -836,7 +828,7 @@ def f(self, other, axis=default_axis, level=None):
             return _combine_series_frame(self, other, op, axis=axis)
         else:
             # in this case we always have `np.ndim(other) == 0`
-            new_data = dispatch_to_series(self, other, op)
+            new_data = dispatch_to_series(self, other, op, str_rep)
             return self._construct_result(new_data)
 
     f.__name__ = op_name
@@ -860,13 +852,15 @@ def f(self, other):
             new_data = dispatch_to_series(self, other, op, str_rep)
 
         elif isinstance(other, ABCSeries):
-            new_data = dispatch_to_series(self, other, op, axis="columns")
+            new_data = dispatch_to_series(
+                self, other, op, str_rep=str_rep, axis="columns"
+            )
 
         else:
 
             # straight boolean comparisons we want to allow all columns
             # (regardless of dtype to pass thru) See #4537 for discussion.
-            new_data = dispatch_to_series(self, other, op)
+            new_data = dispatch_to_series(self, other, op, str_rep)
 
         return self._construct_result(new_data)