BUG: Make dict iterators real iterators, provide "next()" in Python 2

Author: Pietro Battiston <[email protected]>

Closes #12299 from toobaz/compat_iters and squashes the following commits:

1d18284 [Pietro Battiston] BUG: Panels don't behave as dicts (.e.g ".items" is not callable)
2d03e4f [Pietro Battiston] BUG: Make dict iterators real iterators, provide "next()" in Python 2

Author: toobaz

doc/source/whatsnew/v0.18.0.txt

@@ -1128,7 +1128,7 @@ Bug Fixes
 - Work around compat issues with ``boto`` in python 3.5 (:issue:`11915`)
 - Bug in ``NaT`` subtraction from ``Timestamp`` or ``DatetimeIndex`` with timezones (:issue:`11718`)
 - Bug in subtraction of ``Series`` of a single tz-aware ``Timestamp`` (:issue:`12290`)
-
+- Use compat iterators in PY2 to support ``.next()`` (:issue:`12299`)
 - Bug in ``Timedelta.round`` with negative values (:issue:`11690`)
 - Bug in ``.loc`` against ``CategoricalIndex`` may result in normal ``Index`` (:issue:`11586`)
 - Bug in ``DataFrame.info`` when duplicated column names exist (:issue:`11761`)

pandas/compat/__init__.py

@@ -153,31 +153,28 @@ def signature(f):
     lfilter = builtins.filter
 
 
-def iteritems(obj, **kwargs):
-    """replacement for six's iteritems for Python2/3 compat
-       uses 'iteritems' if available and otherwise uses 'items'.
+if PY2:
+    def iteritems(obj, **kw):
+        return obj.iteritems(**kw)
 
-       Passes kwargs to method.
-    """
-    func = getattr(obj, "iteritems", None)
-    if not func:
-        func = obj.items
-    return func(**kwargs)
+    def iterkeys(obj, **kw):
+        return obj.iterkeys(**kw)
 
+    def itervalues(obj, **kw):
+        return obj.itervalues(**kw)
 
-def iterkeys(obj, **kwargs):
-    func = getattr(obj, "iterkeys", None)
-    if not func:
-        func = obj.keys
-    return func(**kwargs)
+    next = lambda it : it.next()
+else:
+    def iteritems(obj, **kw):
+        return iter(obj.items(**kw))
 
+    def iterkeys(obj, **kw):
+        return iter(obj.keys(**kw))
 
-def itervalues(obj, **kwargs):
-    func = getattr(obj, "itervalues", None)
-    if not func:
-        func = obj.values
-    return func(**kwargs)
+    def itervalues(obj, **kw):
+        return iter(obj.values(**kw))
 
+    next = next
 
 def bind_method(cls, name, func):
     """Bind a method to class, python 2 and python 3 compatible.

pandas/core/generic.py

@@ -3119,7 +3119,7 @@ def fillna(self, value=None, method=None, axis=None, inplace=False,
 
                 # fill in 2d chunks
                 result = dict([(col, s.fillna(method=method, value=value))
-                               for col, s in compat.iteritems(self)])
+                               for col, s in self.iteritems()])
                 return self._constructor.from_dict(result).__finalize__(self)
 
             # 2d or less

pandas/core/panel.py

@@ -398,7 +398,7 @@ def to_sparse(self, fill_value=None, kind='block'):
         y : SparseDataFrame
         """
         from pandas.core.sparse import SparsePanel
-        frames = dict(compat.iteritems(self))
+        frames = dict(self.iteritems())
         return SparsePanel(frames, items=self.items,
                            major_axis=self.major_axis,
                            minor_axis=self.minor_axis, default_kind=kind,
@@ -450,7 +450,7 @@ def to_excel(self, path, na_rep='', engine=None, **kwargs):
             writer = path
         kwargs['na_rep'] = na_rep
 
-        for item, df in compat.iteritems(self):
+        for item, df in self.iteritems():
             name = str(item)
             df.to_excel(writer, name, **kwargs)
         writer.save()

pandas/io/pytables.py

@@ -2726,7 +2726,7 @@ def write(self, obj, **kwargs):
         self.attrs.default_kind = obj.default_kind
         self.write_index('items', obj.items)
 
-        for name, sdf in compat.iteritems(obj):
+        for name, sdf in obj.iteritems():
             key = 'sparse_frame_%s' % name
             if key not in self.group._v_children:
                 node = self._handle.create_group(self.group, key)

pandas/sparse/panel.py

@@ -393,7 +393,7 @@ def _combine(self, other, func, axis=0):
             return self._combinePanel(other, func)
         elif lib.isscalar(other):
             new_frames = dict((k, func(v, other))
-                              for k, v in compat.iteritems(self))
+                              for k, v in self.iteritems())
             return self._new_like(new_frames)
 
     def _combineFrame(self, other, func, axis=0):
@@ -470,7 +470,7 @@ def major_xs(self, key):
         y : DataFrame
             index -> minor axis, columns -> items
         """
-        slices = dict((k, v.xs(key)) for k, v in compat.iteritems(self))
+        slices = dict((k, v.xs(key)) for k, v in self.iteritems())
         return DataFrame(slices, index=self.minor_axis, columns=self.items)
 
     def minor_xs(self, key):
@@ -487,7 +487,7 @@ def minor_xs(self, key):
         y : SparseDataFrame
             index -> major axis, columns -> items
         """
-        slices = dict((k, v[key]) for k, v in compat.iteritems(self))
+        slices = dict((k, v[key]) for k, v in self.iteritems())
         return SparseDataFrame(slices, index=self.major_axis,
                                columns=self.items,
                                default_fill_value=self.default_fill_value,

pandas/sparse/tests/test_sparse.py

@@ -104,7 +104,7 @@ def assert_sp_frame_equal(left, right, exact_indices=True):
 
 
 def assert_sp_panel_equal(left, right, exact_indices=True):
-    for item, frame in compat.iteritems(left):
+    for item, frame in left.iteritems():
         assert (item in right)
         # trade-off?
         assert_sp_frame_equal(frame, right[item], exact_indices=exact_indices)

pandas/stats/tests/test_fama_macbeth.py

@@ -44,7 +44,7 @@ def checkFamaMacBethExtended(self, window_type, x, y, **kwds):
             end = index[i + window - 1]
 
             x2 = {}
-            for k, v in compat.iteritems(x):
+            for k, v in x.iteritems():
                 x2[k] = v.truncate(start, end)
             y2 = y.truncate(start, end)
 

pandas/stats/tests/test_ols.py

@@ -573,7 +573,7 @@ def test_wls_panel(self):
 
         stack_y = y.stack()
         stack_x = DataFrame(dict((k, v.stack())
-                                 for k, v in compat.iteritems(x)))
+                                 for k, v in x.iteritems()))
 
         weights = x.std('items')
         stack_weights = weights.stack()

pandas/tests/test_compat.py

@@ -4,7 +4,8 @@
 """
 
 from pandas.compat import (range, zip, map, filter, lrange, lzip, lmap,
-                           lfilter, builtins)
+                           lfilter, builtins, iterkeys, itervalues, iteritems,
+                           next)
 import pandas.util.testing as tm
 
 
@@ -61,3 +62,8 @@ def test_zip(self):
         expected = list(builtins.zip(*lst)),
         lengths = 10,
         self.check_result(actual, expected, lengths)
+
+    def test_dict_iterators(self):
+        self.assertEqual(next(itervalues({1: 2})), 2)
+        self.assertEqual(next(iterkeys({1: 2})), 1)
+        self.assertEqual(next(iteritems({1: 2})), (1, 2))

pandas/tests/test_panel.py

@@ -318,10 +318,10 @@ def test_keys(self):
     def test_iteritems(self):
         # Test panel.iteritems(), aka panel.iteritems()
         # just test that it works
-        for k, v in compat.iteritems(self.panel):
+        for k, v in self.panel.iteritems():
             pass
 
-        self.assertEqual(len(list(compat.iteritems(self.panel))),
+        self.assertEqual(len(list(self.panel.iteritems())),
                          len(self.panel.items))
 
     @ignore_sparse_panel_future_warning
@@ -1105,7 +1105,7 @@ def test_ctor_dict(self):
         assert_panel_equal(result, expected)
 
     def test_constructor_dict_mixed(self):
-        data = dict((k, v.values) for k, v in compat.iteritems(self.panel))
+        data = dict((k, v.values) for k, v in self.panel.iteritems())
         result = Panel(data)
         exp_major = Index(np.arange(len(self.panel.major_axis)))
         self.assertTrue(result.major_axis.equals(exp_major))
@@ -1872,7 +1872,7 @@ def test_shift(self):
         # negative numbers, #2164
         result = self.panel.shift(-1)
         expected = Panel(dict((i, f.shift(-1)[:-1])
-                              for i, f in compat.iteritems(self.panel)))
+                              for i, f in self.panel.iteritems()))
         assert_panel_equal(result, expected)
 
         # mixed dtypes #6959
@@ -2072,7 +2072,7 @@ def test_to_excel(self):
                 except ImportError:
                     raise nose.SkipTest("need xlwt xlrd openpyxl")
 
-                for item, df in compat.iteritems(self.panel):
+                for item, df in self.panel.iteritems():
                     recdf = reader.parse(str(item), index_col=0)
                     assert_frame_equal(df, recdf)
 
@@ -2092,7 +2092,7 @@ def test_to_excel_xlsxwriter(self):
             except ImportError as e:
                 raise nose.SkipTest("cannot write excel file: %s" % e)
 
-            for item, df in compat.iteritems(self.panel):
+            for item, df in self.panel.iteritems():
                 recdf = reader.parse(str(item), index_col=0)
                 assert_frame_equal(df, recdf)
 

pandas/tests/test_panel4d.py

@@ -12,7 +12,6 @@
 from pandas.core.panel4d import Panel4D
 from pandas.core.series import remove_na
 import pandas.core.common as com
-from pandas import compat
 
 from pandas.util.testing import (assert_panel_equal,
                                  assert_panel4d_equal,
@@ -232,7 +231,7 @@ def test_keys(self):
     def test_iteritems(self):
         """Test panel4d.iteritems()"""
 
-        self.assertEqual(len(list(compat.iteritems(self.panel4d))),
+        self.assertEqual(len(list(self.panel4d.iteritems())),
                          len(self.panel4d.labels))
 
     def test_combinePanel4d(self):
@@ -731,7 +730,7 @@ def test_ctor_dict(self):
         # assert_panel_equal(result, expected)
 
     def test_constructor_dict_mixed(self):
-        data = dict((k, v.values) for k, v in compat.iteritems(self.panel4d))
+        data = dict((k, v.values) for k, v in self.panel4d.iteritems())
         result = Panel4D(data)
         exp_major = Index(np.arange(len(self.panel4d.major_axis)))
         self.assertTrue(result.major_axis.equals(exp_major))

pandas/tools/tests/test_merge.py

@@ -2671,7 +2671,7 @@ def test_panel_join_many(self):
 
         data_dict = {}
         for p in panels:
-            data_dict.update(compat.iteritems(p))
+            data_dict.update(p.iteritems())
 
         joined = panels[0].join(panels[1:], how='inner')
         expected = Panel.from_dict(data_dict, intersect=True)