TST: merge test coverage and trim floating point zeros even if there are NAs

Author: wesm

pandas/core/format.py

@@ -488,7 +488,7 @@ def __init__(self, *args, **kwargs):
     def _format_with(self, fmt_str):
         fmt_values = [fmt_str % x if notnull(x) else self.na_rep
                       for x in self.values]
-        return _trim_zeros(fmt_values)
+        return _trim_zeros(fmt_values, self.na_rep)
 
     def get_result(self):
         if self.formatter is not None:
@@ -537,17 +537,22 @@ def just(x):
 
     return [just(x) for x in strings]
 
-def _trim_zeros(str_floats):
+def _trim_zeros(str_floats, na_rep='NaN'):
     """
     Trims zeros and decimal points
     """
     # TODO: what if exponential?
     trimmed = str_floats
-    while len(str_floats) > 0 and all([x.endswith('0') for x in trimmed]):
-        trimmed = [x[:-1] for x in trimmed]
+
+    def _cond(values):
+        non_na = [x for x in values if x != na_rep]
+        return len(non_na) > 0 and all([x.endswith('0') for x in non_na])
+
+    while _cond(trimmed):
+        trimmed = [x[:-1] if x != na_rep else x for x in trimmed]
 
     # trim decimal points
-    return [x[:-1] if x.endswith('.') else x for x in trimmed]
+    return [x[:-1] if x.endswith('.') and x != na_rep else x for x in trimmed]
 
 
 def single_column_table(column):

pandas/tests/test_format.py

@@ -241,12 +241,24 @@ def test_to_string_format_na(self):
                         'B' : [np.nan, 'foo', 'foooo', 'fooooo', 'bar']})
         result = df.to_string()
 
-        expected = ('          A       B\n'
-                    '0       NaN     NaN\n'
-                    '1 -1.000000     foo\n'
-                    '2 -2.123400   foooo\n'
-                    '3  3.000000  fooooo\n'
-                    '4  4.000000     bar')
+        expected = ('        A       B\n'
+                    '0     NaN     NaN\n'
+                    '1 -1.0000     foo\n'
+                    '2 -2.1234   foooo\n'
+                    '3  3.0000  fooooo\n'
+                    '4  4.0000     bar')
+        self.assertEqual(result, expected)
+
+        df = DataFrame({'A' : [np.nan, -1., -2., 3., 4.],
+                        'B' : [np.nan, 'foo', 'foooo', 'fooooo', 'bar']})
+        result = df.to_string()
+
+        expected = ('    A       B\n'
+                    '0 NaN     NaN\n'
+                    '1  -1     foo\n'
+                    '2  -2   foooo\n'
+                    '3   3  fooooo\n'
+                    '4   4     bar')
         self.assertEqual(result, expected)
 
     def test_to_html(self):
@@ -354,11 +366,11 @@ def test_to_string_float_na_spacing(self):
         s[::2] = np.nan
 
         result = s.to_string()
-        expected = ('0         NaN\n'
-                    '1    1.567800\n'
-                    '2         NaN\n'
-                    '3   -3.000000\n'
-                    '4         NaN')
+        expected = ('0       NaN\n'
+                    '1    1.5678\n'
+                    '2       NaN\n'
+                    '3   -3.0000\n'
+                    '4       NaN')
         self.assertEqual(result, expected)
 
 class TestEngFormatter(unittest.TestCase):

pandas/tests/test_frame.py

@@ -8,6 +8,8 @@
 import sys
 import unittest
 
+import nose
+
 from numpy import random, nan
 from numpy.random import randn
 import numpy as np

pandas/tests/test_panel.py

@@ -5,6 +5,7 @@
 import os
 import operator
 import unittest
+import nose
 
 import numpy as np
 
@@ -990,13 +991,20 @@ def test_from_frame_level1_unsorted(self):
         assert_frame_equal(p.minor_xs(2), df.ix[:,2].sort_index())
 
     def test_to_excel(self):
+        try:
+            import xlwt
+            import xlrd
+            import openpyxl
+        except ImportError:
+            raise nose.SkipTest
+
         path = '__tmp__.xlsx'
         self.panel.to_excel(path)
         reader = ExcelFile(path)
         for item, df in self.panel.iteritems():
-            recdf = reader.parse(str(item),index_col=0) 
+            recdf = reader.parse(str(item),index_col=0)
             assert_frame_equal(df, recdf)
-    
+
 class TestLongPanel(unittest.TestCase):
     """
     LongPanel no longer exists, but...

pandas/tools/merge.py

@@ -221,6 +221,12 @@ def _get_merge_keys(self):
                 else:
                     left_keys.append(left[k].values)
                     join_names.append(k)
+            if isinstance(self.right.index, MultiIndex):
+                right_keys = [lev.values.take(lab)
+                              for lev, lab in zip(self.right.index.levels,
+                                                  self.right.index.labels)]
+            else:
+                right_keys = [self.right.index.values]
         elif _any(self.right_on):
             for k in self.right_on:
                 if is_rkey(k):
@@ -229,6 +235,12 @@ def _get_merge_keys(self):
                 else:
                     right_keys.append(right[k].values)
                     join_names.append(k)
+            if isinstance(self.left.index, MultiIndex):
+                left_keys = [lev.values.take(lab)
+                             for lev, lab in zip(self.left.index.levels,
+                                                 self.left.index.labels)]
+            else:
+                left_keys = [self.left.index.values]
 
         if right_drop:
             self.right = self.right.drop(right_drop, axis=1)
@@ -248,7 +260,6 @@ def _validate_specification(self):
             elif self.right_index:
                 if self.left_on is None:
                     raise Exception('Must pass left_on or left_index=True')
-                assert(len(self.left_on) == self.right.index.nlevels)
             else:
                 # use the common columns
                 common_cols = self.left.columns.intersection(self.right.columns)
@@ -261,19 +272,14 @@ def _validate_specification(self):
         elif self.left_on is not None:
             n = len(self.left_on)
             if self.right_index:
+                assert(len(self.left_on) == self.right.index.nlevels)
                 self.right_on = [None] * n
-            else:
-                assert(len(self.right_on) == n)
         elif self.right_on is not None:
             n = len(self.right_on)
             if self.left_index:
+                assert(len(self.right_on) == self.left.index.nlevels)
                 self.left_on = [None] * n
-            else:
-                assert(len(self.left_on) == n)
-        elif self.left_index:
-            assert(len(self.right_on) == self.left.index.nlevels)
-        elif self.right_index:
-            assert(len(self.left_on) == self.right.index.nlevels)
+        assert(len(self.right_on) == len(self.left_on))
 
     def _get_group_keys(self):
         """
@@ -285,25 +291,8 @@ def _get_group_keys(self):
         -------
 
         """
-        if self.left_index:
-            if isinstance(self.left.index, MultiIndex):
-                left_keys = [lev.values.take(lab)
-                             for lev, lab in zip(self.left.index.levels,
-                                                 self.left.index.labels)]
-            else:
-                left_keys = [self.left.index.values]
-        else:
-            left_keys = self.left_join_keys
-
-        if self.right_index:
-            if isinstance(self.right.index, MultiIndex):
-                right_keys = [lev.values.take(lab)
-                              for lev, lab in zip(self.right.index.levels,
-                                                  self.right.index.labels)]
-            else:
-                right_keys = [self.right.index.values]
-        else:
-            right_keys = self.right_join_keys
+        left_keys = self.left_join_keys
+        right_keys = self.right_join_keys
 
         assert(len(left_keys) == len(right_keys))
 

pandas/tools/tests/test_merge.py

@@ -35,29 +35,29 @@ class TestMerge(unittest.TestCase):
 
     def setUp(self):
         # aggregate multiple columns
-        self.df = DataFrame({'key1' : get_test_data(),
-                             'key2' : get_test_data(),
-                             'data1' : np.random.randn(N),
-                             'data2' : np.random.randn(N)})
+        self.df = DataFrame({'key1': get_test_data(),
+                             'key2': get_test_data(),
+                             'data1': np.random.randn(N),
+                             'data2': np.random.randn(N)})
 
         # exclude a couple keys for fun
         self.df = self.df[self.df['key2'] > 1]
 
         self.df2 = DataFrame({'key1'  : get_test_data(n=N//5),
                               'key2'  : get_test_data(ngroups=NGROUPS//2,
                                                       n=N//5),
-                              'value' : np.random.randn(N // 5)})
+                              'value': np.random.randn(N // 5)})
 
         index, data = tm.getMixedTypeDict()
         self.target = DataFrame(data, index=index)
 
         # Join on string value
-        self.source = DataFrame({'MergedA' : data['A'], 'MergedD' : data['D']},
+        self.source = DataFrame({'MergedA': data['A'], 'MergedD': data['D']},
                                 index=data['C'])
 
-        self.left = DataFrame({'key' : ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
-                          'v1' : np.random.randn(7)})
-        self.right = DataFrame({'v2' : np.random.randn(4)},
+        self.left = DataFrame({'key': ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
+                          'v1': np.random.randn(7)})
+        self.right = DataFrame({'v2': np.random.randn(4)},
                            index=['d', 'b', 'c', 'a'])
 
     def test_cython_left_outer_join(self):
@@ -134,9 +134,6 @@ def test_cython_inner_join(self):
         self.assert_(np.array_equal(ls, exp_ls))
         self.assert_(np.array_equal(rs, exp_rs))
 
-    def test_cython_full_outer_join(self):
-        pass
-
     def test_left_outer_join(self):
         joined_key2 = merge(self.df, self.df2, on='key2')
         _check_join(self.df, self.df2, joined_key2, ['key2'], how='left')
@@ -197,11 +194,11 @@ def test_join_on(self):
         self.assert_(np.array_equal(merged['MergedD'], target['D']))
 
         # join with duplicates (fix regression from DataFrame/Matrix merge)
-        df = DataFrame({'key' : ['a', 'a', 'b', 'b', 'c']})
-        df2 = DataFrame({'value' : [0, 1, 2]}, index=['a', 'b', 'c'])
+        df = DataFrame({'key': ['a', 'a', 'b', 'b', 'c']})
+        df2 = DataFrame({'value': [0, 1, 2]}, index=['a', 'b', 'c'])
         joined = df.join(df2, on='key')
-        expected = DataFrame({'key' : ['a', 'a', 'b', 'b', 'c'],
-                              'value' : [0, 0, 1, 1, 2]})
+        expected = DataFrame({'key': ['a', 'a', 'b', 'b', 'c'],
+                              'value': [0, 0, 1, 1, 2]})
         assert_frame_equal(joined, expected)
 
         # Test when some are missing
@@ -245,8 +242,8 @@ def test_join_with_len0(self):
         self.assertEqual(len(merged2), 0)
 
     def test_join_on_inner(self):
-        df = DataFrame({'key' : ['a', 'a', 'd', 'b', 'b', 'c']})
-        df2 = DataFrame({'value' : [0, 1]}, index=['a', 'b'])
+        df = DataFrame({'key': ['a', 'a', 'd', 'b', 'b', 'c']})
+        df2 = DataFrame({'value': [0, 1]}, index=['a', 'b'])
 
         joined = df.join(df2, on='key', how='inner')
 
@@ -257,8 +254,8 @@ def test_join_on_inner(self):
         self.assert_(joined.index.equals(expected.index))
 
     def test_join_on_singlekey_list(self):
-        df = DataFrame({'key' : ['a', 'a', 'b', 'b', 'c']})
-        df2 = DataFrame({'value' : [0, 1, 2]}, index=['a', 'b', 'c'])
+        df = DataFrame({'key': ['a', 'a', 'b', 'b', 'c']})
+        df2 = DataFrame({'value': [0, 1, 2]}, index=['a', 'b', 'c'])
 
         # corner cases
         joined = df.join(df2, on=['key'])
@@ -277,18 +274,18 @@ def test_join_on_series_buglet(self):
         ds = Series([2], index=[1], name='b')
         result = df.join(ds, on='a')
         expected = DataFrame({'a' : [1, 1],
-                              'b' : [2, 2]}, index=df.index)
+                              'b': [2, 2]}, index=df.index)
         tm.assert_frame_equal(result, expected)
 
     def test_join_index_mixed(self):
 
-        df1 = DataFrame({'A' : 1., 'B' : 2, 'C' : 'foo', 'D' : True},
+        df1 = DataFrame({'A': 1., 'B': 2, 'C': 'foo', 'D': True},
                         index=np.arange(10),
                         columns=['A', 'B', 'C', 'D'])
         self.assert_(df1['B'].dtype == np.int64)
         self.assert_(df1['D'].dtype == np.bool_)
 
-        df2 = DataFrame({'A' : 1., 'B' : 2, 'C' : 'foo', 'D' : True},
+        df2 = DataFrame({'A': 1., 'B': 2, 'C': 'foo', 'D': True},
                         index=np.arange(0, 10, 2),
                         columns=['A', 'B', 'C', 'D'])
 
@@ -375,8 +372,8 @@ def test_join_inner_multiindex(self):
                'three', 'one']
 
         data = np.random.randn(len(key1))
-        data = DataFrame({'key1' : key1, 'key2' : key2,
-                         'data' : data})
+        data = DataFrame({'key1': key1, 'key2': key2,
+                         'data': data})
 
         index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
                                    ['one', 'two', 'three']],
@@ -416,9 +413,9 @@ def test_join_float64_float32(self):
         assert_frame_equal(joined, expected)
 
     def test_merge_index_singlekey_right_vs_left(self):
-        left = DataFrame({'key' : ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
-                          'v1' : np.random.randn(7)})
-        right = DataFrame({'v2' : np.random.randn(4)},
+        left = DataFrame({'key': ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
+                          'v1': np.random.randn(7)})
+        right = DataFrame({'v2': np.random.randn(4)},
                            index=['d', 'b', 'c', 'a'])
 
         merged1 = merge(left, right, left_on='key',
@@ -434,9 +431,9 @@ def test_merge_index_singlekey_right_vs_left(self):
         assert_frame_equal(merged1, merged2.ix[:, merged1.columns])
 
     def test_merge_index_singlekey_inner(self):
-        left = DataFrame({'key' : ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
-                          'v1' : np.random.randn(7)})
-        right = DataFrame({'v2' : np.random.randn(4)},
+        left = DataFrame({'key': ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
+                          'v1': np.random.randn(7)})
+        right = DataFrame({'v2': np.random.randn(4)},
                            index=['d', 'b', 'c', 'a'])
 
         # inner join
@@ -459,6 +456,9 @@ def test_merge_misspecified(self):
         self.assertRaises(Exception, merge, self.left, self.left,
                           left_on='key', on='key')
 
+        self.assertRaises(Exception, merge, self.df, self.df2,
+                          left_on=['key1'], right_on=['key1', 'key2'])
+
     def test_merge_overlap(self):
         merged = merge(self.left, self.left, on='key')
         exp_len = (self.left['key'].value_counts() ** 2).sum()
@@ -467,9 +467,9 @@ def test_merge_overlap(self):
         self.assert_('v1.y' in merged)
 
     def test_merge_different_column_key_names(self):
-        left = DataFrame({'lkey' : ['foo', 'bar', 'baz', 'foo'],
-                          'value' : [1, 2, 3, 4]})
-        right = DataFrame({'rkey' : ['foo', 'bar', 'qux', 'foo'],
+        left = DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'],
+                          'value': [1, 2, 3, 4]})
+        right = DataFrame({'rkey': ['foo', 'bar', 'qux', 'foo'],
                            'value' : [5, 6, 7, 8]})
 
         merged = left.merge(right, left_on='lkey', right_on='rkey',
@@ -552,6 +552,13 @@ def test_handle_join_key_pass_array(self):
         self.assert_(np.array_equal(merged['key_0'],
                                     np.array([1, 1, 1, 1, 2, 2, 3, 4, 5])))
 
+        left = DataFrame({'value': range(3)})
+        right = DataFrame({'rvalue' : range(6)})
+
+        key = np.array([0, 1, 1, 2, 2, 3])
+        merged = merge(left, right, left_index=True, right_on=key, how='outer')
+        self.assert_(np.array_equal(merged['key_0'], key))
+
 class TestMergeMulti(unittest.TestCase):
 
     def setUp(self):