Merge PR #200 bringing Python 3 support

* py3-compat-merge:
  All but one test now passing on Python 3.
  Sort out division for in sparse code.
  Use .iterkv() instead of .iteritems(), which gets incorrectly converted to .items() by 2to3.
  Hack so test works on Python 3.
  More changes to support Python 3 better.
  Fix up more tests for Python 3.
  Fixes so more tests pass on Python 3.
  Pandas can now be imported successfully in Python 3.
  Pandas installs on Python 3

Author: wesm

pandas/core/common.py

@@ -1,8 +1,10 @@
 """
 Misc tools for implementing data structures
 """
-
-from cStringIO import StringIO
+try:
+    from io import BytesIO
+except ImportError:   # Python < 2.6
+    from cStringIO import StringIO as BytesIO
 import itertools
 
 from numpy.lib.format import read_array, write_array
@@ -77,13 +79,13 @@ def notnull(obj):
 def _pickle_array(arr):
     arr = arr.view(np.ndarray)
 
-    buf = StringIO()
+    buf = BytesIO()
     write_array(buf, arr)
 
     return buf.getvalue()
 
 def _unpickle_array(bytes):
-    arr = read_array(StringIO(bytes))
+    arr = read_array(BytesIO(bytes))
     return arr
 
 def _take_1d_bool(arr, indexer, out):
@@ -412,7 +414,7 @@ def rands(n):
     """Generates a random alphanumeric string of length *n*"""
     from random import Random
     import string
-    return ''.join(Random().sample(string.letters+string.digits, n))
+    return ''.join(Random().sample(string.ascii_letters+string.digits, n))
 
 def adjoin(space, *lists):
     """
@@ -480,7 +482,11 @@ def __init__(self, seq, key=lambda x:x):
         for value in seq:
             k = key(value)
             self.setdefault(k, []).append(value)
-    __iter__ = dict.iteritems
+    try:
+        __iter__ = dict.iteritems
+    except AttributeError:  # Python 3
+        def __iter__(self):
+            return iter(dict.items(self))
 
 def map_indices_py(arr):
     """

pandas/core/frame.py

@@ -30,6 +30,7 @@
 from pandas.core.internals import BlockManager, make_block, form_blocks
 from pandas.core.series import Series, _is_bool_indexer
 from pandas.util.decorators import deprecate
+from pandas.util import py3compat
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 import pandas._tseries as _tseries
@@ -277,6 +278,10 @@ def iteritems(self):
         """Iterator over (column, series) pairs"""
         series = self._series
         return ((k, series[k]) for k in self.columns)
+    
+    iterkv = iteritems
+    if py3compat.PY3:
+        items = iteritems
 
     def __len__(self):
         """Returns length of index"""
@@ -292,7 +297,7 @@ def __contains__(self, key):
     add = _arith_method(operator.add, 'add')
     mul = _arith_method(operator.mul, 'multiply')
     sub = _arith_method(operator.sub, 'subtract')
-    div = _arith_method(operator.div, 'divide')
+    div = _arith_method(lambda x, y: x / y, 'divide')
 
     radd = _arith_method(operator.add, 'add')
     rmul = _arith_method(operator.mul, 'multiply')
@@ -302,19 +307,26 @@ def __contains__(self, key):
     __add__ = _arith_method(operator.add, '__add__', default_axis=None)
     __sub__ = _arith_method(operator.sub, '__sub__', default_axis=None)
     __mul__ = _arith_method(operator.mul, '__mul__', default_axis=None)
-    __div__ = _arith_method(operator.div, '__div__', default_axis=None)
     __truediv__ = _arith_method(operator.truediv, '__truediv__',
                                default_axis=None)
+    __floordiv__ = _arith_method(operator.floordiv, '__floordiv__',
+                               default_axis=None)
     __pow__ = _arith_method(operator.pow, '__pow__', default_axis=None)
 
     __radd__ = _arith_method(operator.add, '__radd__', default_axis=None)
     __rmul__ = _arith_method(operator.mul, '__rmul__', default_axis=None)
     __rsub__ = _arith_method(lambda x, y: y - x, '__rsub__', default_axis=None)
-    __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__', default_axis=None)
     __rtruediv__ = _arith_method(lambda x, y: y / x, '__rtruediv__',
                                 default_axis=None)
+    __rfloordiv__ = _arith_method(lambda x, y: y // x, '__rfloordiv__',
+                               default_axis=None)
     __rpow__ = _arith_method(lambda x, y: y ** x, '__rpow__',
                              default_axis=None)
+    
+    # Python 2 division methods
+    if not py3compat.PY3:
+        __div__ = _arith_method(operator.div, '__div__', default_axis=None)
+        __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__', default_axis=None)
 
     def __neg__(self):
         return self * -1
@@ -464,7 +476,7 @@ def to_sparse(self, fill_value=None, kind='block'):
                                default_fill_value=fill_value)
 
     def to_csv(self, path, nanRep='', cols=None, header=True,
-              index=True, index_label=None, mode='wb'):
+              index=True, index_label=None, mode='w'):
         """
         Write DataFrame to a comma-separated values (csv) file
 
@@ -483,7 +495,7 @@ def to_csv(self, path, nanRep='', cols=None, header=True,
             Column label for index column(s) if desired. If None is given, and
             `header` and `index` are True, then the index names are used. A
             sequence should be given if the DataFrame uses MultiIndex.
-        mode : Python write mode, default 'wb'
+        mode : Python write mode, default 'w'
         """
         f = open(path, mode)
         csvout = csv.writer(f)
@@ -658,7 +670,7 @@ def dtypes(self):
 
     def get_dtype_counts(self):
         counts = {}
-        for _, series in self.iteritems():
+        for _, series in self.iterkv():
             if series.dtype in counts:
                 counts[series.dtype] += 1
             else:
@@ -915,7 +927,7 @@ def _set_item(self, key, value):
     def _sanitize_column(self, value):
         # Need to make sure new columns (which go into the BlockManager as new
         # blocks) are always copied
-        if hasattr(value, '__iter__'):
+        if hasattr(value, '__iter__') and not isinstance(value, basestring):
             if isinstance(value, Series):
                 if value.index.equals(self.index):
                     # copy the values

pandas/core/panel.py

@@ -15,6 +15,7 @@
 from pandas.core.generic import AxisProperty, NDFrame
 from pandas.core.series import Series
 from pandas.util.decorators import deprecate
+from pandas.util import py3compat
 import pandas.core.common as common
 import pandas._tseries as _tseries
 
@@ -170,16 +171,22 @@ class Panel(NDFrame):
 
     __add__ = _arith_method(operator.add, '__add__')
     __sub__ = _arith_method(operator.sub, '__sub__')
+    __truediv__ = _arith_method(operator.truediv, '__truediv__')
+    __floordiv__ = _arith_method(operator.floordiv, '__floordiv__')
     __mul__ = _arith_method(operator.mul, '__mul__')
-    __div__ = _arith_method(operator.div, '__div__')
     __pow__ = _arith_method(operator.pow, '__pow__')
 
     __radd__ = _arith_method(operator.add, '__radd__')
     __rmul__ = _arith_method(operator.mul, '__rmul__')
     __rsub__ = _arith_method(lambda x, y: y - x, '__rsub__')
-    __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__')
+    __rtruediv__ = _arith_method(lambda x, y: y / x, '__rtruediv__')
+    __rfloordiv__ = _arith_method(lambda x, y: y // x, '__rfloordiv__')
     __rpow__ = _arith_method(lambda x, y: y ** x, '__rpow__')
 
+    if not py3compat.PY3:
+        __div__ = _arith_method(operator.div, '__div__')
+        __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__')
+
     def __init__(self, data, items=None, major_axis=None, minor_axis=None,
                  copy=False, dtype=None):
         """
@@ -329,6 +336,10 @@ def __iter__(self):
     def iteritems(self):
         for item in self.items:
             yield item, self[item]
+    
+    # Name that won't get automatically converted to items by 2to3. items is
+    # already in use for the first axis.
+    iterkv = iteritems
 
     def _get_plane_axes(self, axis):
         """
@@ -383,7 +394,7 @@ def to_sparse(self, fill_value=None, kind='block'):
         y : SparseDataFrame
         """
         from pandas.core.sparse import SparsePanel
-        frames = dict(self.iteritems())
+        frames = dict(self.iterkv())
         return SparsePanel(frames, items=self.items,
                            major_axis=self.major_axis,
                            minor_axis=self.minor_axis,
@@ -632,7 +643,7 @@ def fillna(self, value=None, method='pad'):
         """
         if value is None:
             result = {}
-            for col, s in self.iteritems():
+            for col, s in self.iterkv():
                 result[col] = s.fillna(method=method, value=value)
 
             return Panel.from_dict(result)
@@ -642,8 +653,12 @@ def fillna(self, value=None, method='pad'):
 
     add = _panel_arith_method(operator.add, 'add')
     subtract = sub = _panel_arith_method(operator.sub, 'subtract')
-    divide = div = _panel_arith_method(operator.div, 'divide')
     multiply = mul = _panel_arith_method(operator.mul, 'multiply')
+    
+    try:
+        divide = div = _panel_arith_method(operator.div, 'divide')
+    except AttributeError:   # Python 3
+        divide = div = _panel_arith_method(operator.truediv, 'divide')
 
     def major_xs(self, key, copy=True):
         """
@@ -1214,8 +1229,12 @@ def _combine_panel_frame(self, other, func, axis='items'):
 
     add = _panel_arith_method(operator.add, 'add')
     subtract = sub = _panel_arith_method(operator.sub, 'subtract')
-    divide = div = _panel_arith_method(operator.div, 'divide')
     multiply = mul = _panel_arith_method(operator.mul, 'multiply')
+    
+    try:
+        divide = div = _panel_arith_method(operator.div, 'divide')
+    except AttributeError:   # Python 3
+        divide = div = _panel_arith_method(operator.truediv, 'divide')
 
     def to_wide(self):
         """

pandas/core/series.py

@@ -21,6 +21,7 @@
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import _SeriesIndexer, _maybe_droplevels
 from pandas.util.decorators import deprecate
+from pandas.util import py3compat
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 import pandas._tseries as lib
@@ -418,31 +419,41 @@ def iteritems(self):
         Lazily iterate over (index, value) tuples
         """
         return itertools.izip(iter(self.index), iter(self))
+    
+    iterkv = iteritems
+    if py3compat.PY3:
+        items = iteritems
 
     #----------------------------------------------------------------------
     #   Arithmetic operators
 
     __add__ = _arith_method(operator.add, '__add__')
     __sub__ = _arith_method(operator.sub, '__sub__')
     __mul__ = _arith_method(operator.mul, '__mul__')
-    __div__ = _arith_method(operator.div, '__div__')
     __truediv__ = _arith_method(operator.truediv, '__truediv__')
+    __floordiv__ = _arith_method(operator.floordiv, '__floordiv__')
     __pow__ = _arith_method(operator.pow, '__pow__')
-    __truediv__ = _arith_method(operator.truediv, '__truediv__')
 
     __radd__ = _arith_method(operator.add, '__add__')
     __rmul__ = _arith_method(operator.mul, '__mul__')
     __rsub__ = _arith_method(lambda x, y: y - x, '__sub__')
-    __rdiv__ = _arith_method(lambda x, y: y / x, '__div__')
     __rtruediv__ = _arith_method(lambda x, y: y / x, '__truediv__')
+    __rfloordiv__ = _arith_method(lambda x, y: y // x, '__floordiv__')
     __rpow__ = _arith_method(lambda x, y: y ** x, '__pow__')
 
     # Inplace operators
     __iadd__ = __add__
     __isub__ = __sub__
     __imul__ = __mul__
-    __idiv__ = __div__
+    __itruediv__ = __truediv__
+    __ifloordiv__ = __floordiv__
     __ipow__ = __pow__
+    
+    # Python 2 division operators
+    if not py3compat.PY3:
+        __div__ = _arith_method(operator.div, '__div__')
+        __rdiv__ = _arith_method(lambda x, y: y / x, '__div__')
+        __idiv__ = __div__
 
     #----------------------------------------------------------------------
     # Misc public methods
@@ -1031,7 +1042,10 @@ def _binop(self, other, func, fill_value=None):
     add = _flex_method(operator.add, 'add')
     sub = _flex_method(operator.sub, 'subtract')
     mul = _flex_method(operator.mul, 'multiply')
-    div = _flex_method(operator.div, 'divide')
+    try:
+        div = _flex_method(operator.div, 'divide')
+    except AttributeError:    # Python 3
+        div = _flex_method(operator.truediv, 'divide')
 
     def combine(self, other, func, fill_value=nan):
         """

pandas/core/sparse.py

@@ -20,6 +20,8 @@
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 
+from pandas.util import py3compat
+
 from pandas._sparse import BlockIndex, IntIndex
 import pandas._sparse as splib
 
@@ -296,24 +298,31 @@ def __repr__(self):
     __add__ = _sparse_op_wrap(operator.add, 'add')
     __sub__ = _sparse_op_wrap(operator.sub, 'sub')
     __mul__ = _sparse_op_wrap(operator.mul, 'mul')
-    __div__ = _sparse_op_wrap(operator.div, 'div')
     __truediv__ = _sparse_op_wrap(operator.truediv, 'truediv')
+    __floordiv__ = _sparse_op_wrap(operator.floordiv, 'floordiv')
     __pow__ = _sparse_op_wrap(operator.pow, 'pow')
 
     # reverse operators
     __radd__ = _sparse_op_wrap(operator.add, '__radd__')
-    __rmul__ = _sparse_op_wrap(operator.mul, '__rmul__')
     __rsub__ = _sparse_op_wrap(lambda x, y: y - x, '__rsub__')
-    __rdiv__ = _sparse_op_wrap(lambda x, y: y / x, '__rdiv__')
+    __rmul__ = _sparse_op_wrap(operator.mul, '__rmul__')
     __rtruediv__ = _sparse_op_wrap(lambda x, y: y / x, '__rtruediv__')
+    __rfloordiv__ = _sparse_op_wrap(lambda x, y: y // x, 'floordiv')
     __rpow__ = _sparse_op_wrap(lambda x, y: y ** x, '__rpow__')
 
     # Inplace operators
     __iadd__ = __add__
     __isub__ = __sub__
     __imul__ = __mul__
-    __idiv__ = __div__
+    __itruediv__ = __truediv__
+    __ifloordiv__ = __floordiv__
     __ipow__ = __pow__
+    
+    # Python 2 division operators
+    if not py3compat.PY3:
+        __div__ = _sparse_op_wrap(operator.div, 'div')
+        __rdiv__ = _sparse_op_wrap(lambda x, y: y / x, '__rdiv__')
+        __idiv__ = __div__
 
     @property
     def values(self):
@@ -1590,7 +1599,7 @@ def _combine(self, other, func, axis=0):
             return self._combinePanel(other, func)
         elif np.isscalar(other):
             new_frames = dict((k, func(v, other))
-                              for k, v in self.iteritems())
+                              for k, v in self.iterkv())
             return self._new_like(new_frames)
 
     def _combineFrame(self, other, func, axis=0):
@@ -1666,7 +1675,7 @@ def major_xs(self, key):
         y : DataFrame
             index -> minor axis, columns -> items
         """
-        slices = dict((k, v.xs(key)) for k, v in self.iteritems())
+        slices = dict((k, v.xs(key)) for k, v in self.iterkv())
         return DataFrame(slices, index=self.minor_axis, columns=self.items)
 
     def minor_xs(self, key):
@@ -1683,7 +1692,7 @@ def minor_xs(self, key):
         y : SparseDataFrame
             index -> major axis, columns -> items
         """
-        slices = dict((k, v[key]) for k, v in self.iteritems())
+        slices = dict((k, v[key]) for k, v in self.iterkv())
         return SparseDataFrame(slices, index=self.major_axis,
                                columns=self.items,
                                default_fill_value=self.default_fill_value,