Merge pull request #11892 from jreback/ri

ENH: RangeIndex redux

Author: jreback

doc/source/advanced.rst

@@ -617,10 +617,20 @@ faster than fancy indexing.
    timeit ser.ix[indexer]
    timeit ser.take(indexer)
 
+.. _indexing.index_types:
+
+Index Types
+-----------
+
+We have discussed ``MultiIndex`` in the previous sections pretty extensively. ``DatetimeIndex`` and ``PeriodIndex``
+are shown :ref:`here <timeseries.overview>`. ``TimedeltaIndex`` are :ref:`here <timedeltas.timedeltas>`.
+
+In the following sub-sections we will highlite some other index types.
+
 .. _indexing.categoricalindex:
 
 CategoricalIndex
-----------------
+~~~~~~~~~~~~~~~~
 
 .. versionadded:: 0.16.1
 
@@ -702,10 +712,21 @@ values NOT in the categories, similarly to how you can reindex ANY pandas index.
       In [12]: pd.concat([df2, df3]
       TypeError: categories must match existing categories when appending
 
+.. _indexing.rangeindex:
+
+Int64Index and RangeIndex
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``Int64Index`` is a fundamental basic index in *pandas*. This is an Immutable array implementing an ordered, sliceable set.
+Prior to 0.18.0, the ``Int64Index`` would provide the default index for all ``NDFrame`` objects.
+
+``RangeIndex`` is a sub-class of ``Int64Index`` added in version 0.18.0, now providing the default index for all ``NDFrame`` objects.
+``RangeIndex`` is an optimized version of ``Int64Index`` that can represent a monotonic ordered set. These are analagous to python :ref:`range types <https://docs.python.org/3/library/stdtypes.html#typesseq-range>`.
+
 .. _indexing.float64index:
 
 Float64Index
-------------
+~~~~~~~~~~~~
 
 .. note::
 

doc/source/timeseries.rst

@@ -1091,7 +1091,7 @@ An example of how holidays and holiday calendars are defined:
 Using this calendar, creating an index or doing offset arithmetic skips weekends
 and holidays (i.e., Memorial Day/July 4th).  For example, the below defines
 a custom business day offset using the ``ExampleCalendar``.  Like any other offset,
-it can be used to create a ``DatetimeIndex`` or added to ``datetime`` 
+it can be used to create a ``DatetimeIndex`` or added to ``datetime``
 or ``Timestamp`` objects.
 
 .. ipython:: python

doc/source/whatsnew/v0.18.0.txt

@@ -19,6 +19,7 @@ Highlights include:
 
 - Window functions are now methods on ``.groupby`` like objects, see :ref:`here <whatsnew_0180.enhancements.moments>`.
 - ``pd.test()`` top-level nose test runner is available (:issue:`4327`)
+- Adding support for a ``RangeIndex`` as a specialized form of the ``Int64Index`` for memory savings, see :ref:`here <whatsnew_0180.enhancements.rangeindex>`.
 
 Check the :ref:`API Changes <whatsnew_0180.api>` and :ref:`deprecations <whatsnew_0180.deprecations>` before updating.
 
@@ -102,6 +103,39 @@ And multiple aggregations
    r.agg({'A' : ['mean','std'],
           'B' : ['mean','std']})
 
+.. _whatsnew_0180.enhancements.rangeindex:
+
+Range Index
+^^^^^^^^^^^
+
+A ``RangeIndex`` has been added to the ``Int64Index`` sub-classes to support a memory saving alternative for common use cases. This has a similar implementation to the python ``range`` object (``xrange`` in python 2), in that it only stores the start, stop, and step values for the index. It will transparently interact with the user API, converting to ``Int64Index`` if needed.
+
+This will now be the default constructed index for ``NDFrame`` objects, rather than previous an ``Int64Index``. (:issue:`939`)
+
+Previous Behavior:
+
+.. code-block:: python
+
+   In [3]: s = Series(range(1000))
+
+   In [4]: s.index
+   Out[4]:
+   Int64Index([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,
+               ...
+               990, 991, 992, 993, 994, 995, 996, 997, 998, 999], dtype='int64', length=1000)
+
+   In [6]: s.index.nbytes
+   Out[6]: 8000
+
+
+New Behavior:
+
+.. ipython:: python
+
+   s = Series(range(1000))
+   s.index
+   s.index.nbytes
+
 .. _whatsnew_0180.enhancements.other:
 
 Other enhancements

pandas/core/api.py

@@ -8,7 +8,8 @@
 from pandas.core.categorical import Categorical
 from pandas.core.groupby import Grouper
 from pandas.core.format import set_eng_float_format
-from pandas.core.index import Index, CategoricalIndex, Int64Index, Float64Index, MultiIndex
+from pandas.core.index import (Index, CategoricalIndex, Int64Index,
+                               RangeIndex, Float64Index, MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/core/common.py

@@ -84,6 +84,8 @@ def _check(cls, inst):
 ABCIndex = create_pandas_abc_type("ABCIndex", "_typ", ("index", ))
 ABCInt64Index = create_pandas_abc_type("ABCInt64Index", "_typ",
                                        ("int64index", ))
+ABCRangeIndex = create_pandas_abc_type("ABCRangeIndex", "_typ",
+                                       ("rangeindex", ))
 ABCFloat64Index = create_pandas_abc_type("ABCFloat64Index", "_typ",
                                          ("float64index", ))
 ABCMultiIndex = create_pandas_abc_type("ABCMultiIndex", "_typ",
@@ -97,7 +99,8 @@ def _check(cls, inst):
 ABCCategoricalIndex = create_pandas_abc_type("ABCCategoricalIndex", "_typ",
                                              ("categoricalindex", ))
 ABCIndexClass = create_pandas_abc_type("ABCIndexClass", "_typ",
-                                       ("index", "int64index", "float64index",
+                                       ("index", "int64index", "rangeindex",
+                                        "float64index",
                                         "multiindex", "datetimeindex",
                                         "timedeltaindex", "periodindex",
                                         "categoricalindex"))
@@ -1805,11 +1808,8 @@ def is_bool_indexer(key):
 
 
 def _default_index(n):
-    from pandas.core.index import Int64Index
-    values = np.arange(n, dtype=np.int64)
-    result = Int64Index(values, name=None)
-    result.is_unique = True
-    return result
+    from pandas.core.index import RangeIndex
+    return RangeIndex(0, n, name=None)
 
 
 def ensure_float(arr):

pandas/core/dtypes.py

@@ -214,5 +214,6 @@ def __eq__(self, other):
         if isinstance(other, compat.string_types):
             return other == self.name
 
-        return isinstance(other, DatetimeTZDtype) and self.unit == other.unit \
-            and self.tz == other.tz
+        return isinstance(other, DatetimeTZDtype) and \
+            self.unit == other.unit and \
+            str(self.tz) == str(other.tz)

pandas/core/frame.py

@@ -5325,7 +5325,7 @@ def extract_index(data):
                            (lengths[0], len(index)))
                     raise ValueError(msg)
             else:
-                index = Index(np.arange(lengths[0]))
+                index = _default_index(lengths[0])
 
     return _ensure_index(index)
 
@@ -5538,11 +5538,11 @@ def convert(arr):
 
 
 def _get_names_from_index(data):
-    index = lrange(len(data))
     has_some_name = any([getattr(s, 'name', None) is not None for s in data])
     if not has_some_name:
-        return index
+        return _default_index(len(data))
 
+    index = lrange(len(data))
     count = 0
     for i, s in enumerate(data):
         n = getattr(s, 'name', None)