Fix irregular Timestamp arithmetic types #6543

doc/source/release.rst

@@ -107,6 +107,10 @@ API Changes
   or numbering columns as needed (:issue:`2385`)
 - Slicing and advanced/boolean indexing operations on ``Index`` classes will no
   longer change type of the resulting index (:issue:`6440`).
+- ``set_index`` no longer converts MultiIndexes to an Index of tuples (:issue:`6459`).
+- Slicing with negative start, stop & step values handles corner cases better (:issue:`6531`):
+  - ``df.iloc[:-len(df)]`` is now empty
+  - ``df.iloc[len(df)::-1]`` now enumerates all elements in reverse
 
 Experimental Features
 ~~~~~~~~~~~~~~~~~~~~~
@@ -139,6 +143,7 @@ Improvements to existing features
 Bug Fixes
 ~~~~~~~~~
 
+- Bug in Series ValueError when index doesn't match data (:issue:`6532`)
 - Bug in ``pd.DataFrame.sort_index`` where mergesort wasn't stable when ``ascending=False`` (:issue:`6399`)
 - Bug in ``pd.tseries.frequencies.to_offset`` when argument has leading zeroes (:issue:`6391`)
 - Bug in version string gen. for dev versions with shallow clones / install from tarball (:issue:`6127`)
@@ -180,7 +185,7 @@ Bug Fixes
 - Bug in :meth:`DataFrame.replace` where nested dicts were erroneously
   depending on the order of dictionary keys and values (:issue:`5338`).
 - Perf issue in concatting with empty objects (:issue:`3259`)
-- Clarify sorting of ``sym_diff`` on ``Index``es with ``NaN``s (:isssue:`6444`)
+- Clarify sorting of ``sym_diff`` on ``Index``es with ``NaN``s (:issue:`6444`)
 - Regression in ``MultiIndex.from_product`` with a ``DatetimeIndex`` as input (:issue:`6439`)
 - Bug in ``str.extract`` when passed a non-default index (:issue:`6348`)
 - Bug in ``str.split`` when passed ``pat=None`` and ``n=1`` (:issue:`6466`)
@@ -194,6 +199,16 @@ Bug Fixes
 - Bug in ``read_html`` tests where redirected invalid URLs would make one test
   fail (:issue:`6445`).
 - Bug in multi-axis indexing using ``.loc`` on non-unique indices (:issue:`6504`)
+- Bug that caused _ref_locs corruption when slice indexing across columns axis of a DataFrame (:issue:`6525`)
+- Regression from 0.13 in the treatmenet of numpy ``datetime64`` non-ns dtypes in Series creation (:issue:`6529`)
+- ``.names`` attribute of MultiIndexes passed to ``set_index`` are now preserved (:issue:`6459`).
+- Bug in setitem with a duplicate index and an alignable rhs (:issue:`6541`)
+- Bug in setitem with loc on mixed integer Indexes (:issue:`6546`)
+- Bug in ``pd.read_stata`` which would use the wrong data types and missing values (:issue:`6327`)
+- Bug in ``DataFrame.to_stata`` that lead to data loss in certain cases, and could exported using the
+  wrong data types and missing values (:issue:`6335`)
+- Inconsistent types in Timestamp addition/subtraction (:issue:`6543`)
+
 
 pandas 0.13.1
 -------------

doc/source/v0.14.0.txt

@@ -92,6 +92,49 @@ These are out-of-bounds selections
   .. ipython:: python
 
      i[[0,1,2]].astype(np.int_)
+- ``set_index`` no longer converts MultiIndexes to an Index of tuples. For example,
+  the old behavior returned an Index in this case (:issue:`6459`):
+
+  .. ipython:: python
+     :suppress:
+
+     from itertools import product
+     tuples = list(product(('a', 'b'), ('c', 'd')))
+     mi = MultiIndex.from_tuples(tuples)
+     df_multi = DataFrame(np.random.randn(4, 2), index=mi)
+     tuple_ind = pd.Index(tuples)
+
+  .. ipython:: python
+
+     df_multi.index
+
+     @suppress
+     df_multi.index = tuple_ind
+
+     # Old behavior, casted MultiIndex to an Index
+     df_multi.set_index(df_multi.index)
+
+     @suppress
+     df_multi.index = mi
+
+     # New behavior
+     df_multi.set_index(df_multi.index)
+
+  This also applies when passing multiple indices to ``set_index``:
+
+  .. ipython:: python
+
+    @suppress
+    df_multi.index = tuple_ind
+
+    # Old output, 2-level MultiIndex of tuples
+    df_multi.set_index([df_multi.index, df_multi.index])
+
+    @suppress
+    df_multi.index = mi
+
+    # New output, 4-level MultiIndex
+    df_multi.set_index([df_multi.index, df_multi.index])
 
 
 MultiIndexing Using Slicers
@@ -248,6 +291,9 @@ Enhancements
   using ``DataFrame.to_csv`` (:issue:`5414`, :issue:`4528`)
 - Added a ``to_julian_date`` function to ``TimeStamp`` and ``DatetimeIndex``
   to convert to the Julian Date used primarily in astronomy. (:issue:`4041`)
+- ``DataFrame.to_stata`` will now check data for compatibility with Stata data types
+  and will upcast when needed.  When it isn't possibly to losslessly upcast, a warning
+  is raised (:issue:`6327`)
 
 Performance
 ~~~~~~~~~~~

pandas/core/common.py

@@ -124,7 +124,7 @@ def isnull(obj):
 
     See also
     --------
-    pandas.notnull: boolean inverse of pandas.isnull    
+    pandas.notnull: boolean inverse of pandas.isnull
     """
     return _isnull(obj)
 
@@ -272,7 +272,7 @@ def notnull(obj):
     isnulled : array-like of bool or bool
         Array or bool indicating whether an object is *not* null or if an array
         is given which of the element is *not* null.
-    
+
     See also
     --------
     pandas.isnull : boolean inverse of pandas.notnull
@@ -1727,10 +1727,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
             dtype = value.dtype
 
             if dtype.kind == 'M' and dtype != _NS_DTYPE:
-                try:
-                    value = tslib.array_to_datetime(value)
-                except:
-                    raise
+                value = value.astype(_NS_DTYPE)
 
             elif dtype.kind == 'm' and dtype != _TD_DTYPE:
                 from pandas.tseries.timedeltas import \

pandas/core/frame.py

@@ -1867,11 +1867,6 @@ def eval(self, expr, **kwargs):
         kwargs['resolvers'] = kwargs.get('resolvers', ()) + resolvers
         return _eval(expr, **kwargs)
 
-    def _slice(self, slobj, axis=0, raise_on_error=False, typ=None):
-        axis = self._get_block_manager_axis(axis)
-        new_data = self._data.get_slice(
-            slobj, axis=axis, raise_on_error=raise_on_error)
-        return self._constructor(new_data)
 
     def _box_item_values(self, key, values):
         items = self.columns[self.columns.get_loc(key)]
@@ -2240,7 +2235,15 @@ def set_index(self, keys, drop=True, append=False, inplace=False,
 
         to_remove = []
         for col in keys:
-            if isinstance(col, Series):
+            if isinstance(col, MultiIndex):
+                # append all but the last column so we don't have to modify
+                # the end of this loop
+                for n in range(col.nlevels - 1):
+                    arrays.append(col.get_level_values(n))
+
+                level = col.get_level_values(col.nlevels - 1)
+                names.extend(col.names)
+            elif isinstance(col, (Series, Index)):
                 level = col.values
                 names.append(col.name)
             elif isinstance(col, (list, np.ndarray)):

pandas/core/generic.py

@@ -1079,6 +1079,16 @@ def _clear_item_cache(self, i=None):
         else:
             self._item_cache.clear()
 
+    def _slice(self, slobj, axis=0, typ=None):
+        """
+        Construct a slice of this container.
+
+        typ parameter is maintained for compatibility with Series slicing.
+
+        """
+        axis = self._get_block_manager_axis(axis)
+        return self._constructor(self._data.get_slice(slobj, axis=axis))
+
     def _set_item(self, key, value):
         self._data.set(key, value)
         self._clear_item_cache()

pandas/core/index.py

@@ -555,6 +555,29 @@ def _convert_list_indexer(self, key, typ=None):
         """ convert a list indexer. these should be locations """
         return key
 
+    def _convert_list_indexer_for_mixed(self, keyarr, typ=None):
+        """ passed a key that is tuplesafe that is integer based
+            and we have a mixed index (e.g. number/labels). figure out
+            the indexer. return None if we can't help
+        """
+        if com.is_integer_dtype(keyarr) and not self.is_floating():
+            if self.inferred_type != 'integer':
+                keyarr = np.where(keyarr < 0,
+                                  len(self) + keyarr, keyarr)
+
+            if self.inferred_type == 'mixed-integer':
+                indexer = self.get_indexer(keyarr)
+                if (indexer >= 0).all():
+                    return indexer
+
+                from pandas.core.indexing import _maybe_convert_indices
+                return _maybe_convert_indices(indexer, len(self))
+
+            elif not self.inferred_type == 'integer':
+                return keyarr
+
+        return None
+
     def _convert_indexer_error(self, key, msg=None):
         if msg is None:
             msg = 'label'
@@ -987,8 +1010,13 @@ def intersection(self, other):
             except TypeError:
                 pass
 
-        indexer = self.get_indexer(other.values)
-        indexer = indexer.take((indexer != -1).nonzero()[0])
+        try:
+            indexer = self.get_indexer(other.values)
+            indexer = indexer.take((indexer != -1).nonzero()[0])
+        except:
+            # duplicates
+            indexer = self.get_indexer_non_unique(other.values)[0].unique()
+
         return self.take(indexer)
 
     def diff(self, other):

pandas/core/indexing.py

@@ -91,32 +91,8 @@ def _get_label(self, label, axis=0):
     def _get_loc(self, key, axis=0):
         return self.obj._ixs(key, axis=axis)
 
-    def _slice(self, obj, axis=0, raise_on_error=False, typ=None):
-
-        # make out-of-bounds into bounds of the object
-        if typ == 'iloc':
-            ax = self.obj._get_axis(axis)
-            l = len(ax)
-            start = obj.start
-            stop = obj.stop
-            step = obj.step
-            if start is not None:
-                # degenerate to return nothing
-                if start >= l:
-                    return self._getitem_axis(tuple(),axis=axis)
-
-                # equiv to a null slice
-                elif start <= -l:
-                    start = None
-            if stop is not None:
-                if stop > l:
-                    stop = None
-                elif stop <= -l:
-                    stop = None
-            obj = slice(start,stop,step)
-
-        return self.obj._slice(obj, axis=axis, raise_on_error=raise_on_error,
-                               typ=typ)
+    def _slice(self, obj, axis=0, typ=None):
+        return self.obj._slice(obj, axis=axis, typ=typ)
 
     def __setitem__(self, key, value):
 
@@ -441,7 +417,9 @@ def can_do_equal_len():
                         # align to
                         if item in value:
                             v = value[item]
-                            v = v.reindex(self.obj[item].index & v.index)
+                            i = self.obj[item].index
+                            v = v.reindex(i & v.index)
+
                             setter(item, v.values)
                         else:
                             setter(item, np.nan)
@@ -909,20 +887,10 @@ def _reindex(keys, level=None):
                 # asarray can be unsafe, NumPy strings are weird
                 keyarr = _asarray_tuplesafe(key)
 
-            if is_integer_dtype(keyarr) and not labels.is_floating():
-                if labels.inferred_type != 'integer':
-                    keyarr = np.where(keyarr < 0,
-                                      len(labels) + keyarr, keyarr)
-
-                if labels.inferred_type == 'mixed-integer':
-                    indexer = labels.get_indexer(keyarr)
-                    if (indexer >= 0).all():
-                        self.obj.take(indexer, axis=axis, convert=True)
-                    else:
-                        return self.obj.take(keyarr, axis=axis)
-                elif not labels.inferred_type == 'integer':
-
-                    return self.obj.take(keyarr, axis=axis)
+            # handle a mixed integer scenario
+            indexer = labels._convert_list_indexer_for_mixed(keyarr, typ=self.name)
+            if indexer is not None:
+                return self.obj.take(indexer, axis=axis)
 
             # this is not the most robust, but...
             if (isinstance(labels, MultiIndex) and
@@ -1062,11 +1030,9 @@ def _convert_to_indexer(self, obj, axis=0, is_setter=False):
                     objarr = _asarray_tuplesafe(obj)
 
                 # If have integer labels, defer to label-based indexing
-                if is_integer_dtype(objarr) and not is_int_index:
-                    if labels.inferred_type != 'integer':
-                        objarr = np.where(objarr < 0,
-                                          len(labels) + objarr, objarr)
-                    return objarr
+                indexer = labels._convert_list_indexer_for_mixed(objarr, typ=self.name)
+                if indexer is not None:
+                    return indexer
 
                 # this is not the most robust, but...
                 if (isinstance(labels, MultiIndex) and
@@ -1353,8 +1319,7 @@ def _get_slice_axis(self, slice_obj, axis=0):
             return obj
 
         if isinstance(slice_obj, slice):
-            return self._slice(slice_obj, axis=axis, raise_on_error=True,
-                               typ='iloc')
+            return self._slice(slice_obj, axis=axis, typ='iloc')
         else:
             return self.obj.take(slice_obj, axis=axis, convert=False)
 
@@ -1657,18 +1622,6 @@ def _need_slice(obj):
             (obj.step is not None and obj.step != 1))
 
 
-def _check_slice_bounds(slobj, values):
-    l = len(values)
-    start = slobj.start
-    if start is not None:
-        if start < -l or start > l - 1:
-            raise IndexError("out-of-bounds on slice (start)")
-    stop = slobj.stop
-    if stop is not None:
-        if stop < -l - 1 or stop > l:
-            raise IndexError("out-of-bounds on slice (end)")
-
-
 def _maybe_droplevels(index, key):
     # drop levels
     original_index = index