Add more tests and fix some existing ones

Author: hexgnu

pandas/core/internals.py

@@ -5523,7 +5523,10 @@ def is_na(self):
     def get_reindexed_values(self, empty_dtype, upcasted_na):
         if upcasted_na is None:
             # No upcasting is necessary
-            fill_value = self.block.fill_value
+
+            # You would think that you want self.block.fill_value here
+            # But in reality that will fill with a bunch of wrong values
+            fill_value = np.nan
             values = self.block.get_values()
         else:
             fill_value = upcasted_na

pandas/core/sparse/frame.py

@@ -73,6 +73,10 @@ def __init__(self, data=None, index=None, columns=None, default_kind=None,
             if columns is None:
                 raise Exception("cannot pass a series w/o a name or columns")
             data = {columns[0]: data}
+        elif isinstance(data, BlockManager):
+            fill_value_size = len(set(b.fill_value for b in data.blocks))
+            if default_fill_value is None and fill_value_size == 1:
+                default_fill_value = data.blocks[0].fill_value
 
         if default_fill_value is None:
             default_fill_value = np.nan

pandas/core/sparse/series.py

@@ -544,7 +544,7 @@ def to_dense(self, sparse_only=False):
             index = self.index.take(int_index.indices)
             return Series(self.sp_values, index=index, name=self.name)
         else:
-            return Series(self.values.to_dense(), index=self.index,
+            return Series(self.get_values(), index=self.index,
                           name=self.name)
 
     @property

pandas/tests/reshape/merge/test_merge.py

@@ -1805,7 +1805,9 @@ def test_merge_on_indexes(self, left_df, right_df, how, sort, expected):
 
 class TestMergeSparseDataFrames(object):
     # Cannot seem to get 0 or 1 working with sparse data frame
-    @pytest.mark.parametrize('fill_value,how', itertools.product([np.nan],
+    @pytest.mark.parametrize('fill_value,how', itertools.product([0, 1,
+                                                                  None,
+                                                                  np.nan],
                                                                  ['left',
                                                                   'right',
                                                                   'outer',
@@ -1816,6 +1818,57 @@ def test_merge_two_sparse_frames(self, fill_value, how):
         dense_threes = pd.DataFrame({'A': list(range(0, 300, 3)),
                                      'B': np.random.randint(0, 100, size=100)})
 
+        sparse_evens = dense_evens.to_sparse(fill_value=fill_value)
+        sparse_threes = dense_threes.to_sparse(fill_value=fill_value)
+
+        to_merge_sparse = [sparse_evens, sparse_threes]
+
+        to_merge_dense = [dense_evens, dense_threes]
+
+        for _ in range(2):
+            sparse_merge = to_merge_sparse[0].merge(to_merge_sparse[1],
+                                                    how=how, on='A')
+
+            dense_merge = to_merge_dense[0].merge(to_merge_dense[1],
+                                                  how=how, on='A')
+
+            # If you merge two dense frames together it tends to default to
+            # float64 not the original dtype
+            dense_merge['B_x'] = dense_merge['B_x'].astype(np.int64,
+                                                           errors='ignore')
+            dense_merge['B_y'] = dense_merge['B_y'].astype(np.int64,
+                                                           errors='ignore')
+
+            if fill_value is None or fill_value is np.nan:
+                assert sparse_merge.default_fill_value is np.nan
+            else:
+                tm.assert_almost_equal(sparse_merge.default_fill_value,
+                                       fill_value)
+
+            exp = dense_merge.to_sparse(fill_value=fill_value),
+            tm.assert_sp_frame_equal(sparse_merge, exp,
+                                     exact_indices=False,
+                                     check_dtype=False)
+
+            to_merge_sparse = to_merge_sparse[::-1]
+            to_merge_dense = to_merge_dense[::-1]
+
+    @pytest.mark.parametrize('fill_value,how', itertools.product([0, 1,
+                                                                  None,
+                                                                  np.nan],
+                                                                 ['left',
+                                                                  'right',
+                                                                  'outer',
+                                                                  'inner']))
+    def test_merge_dense_sparse_frames(self, fill_value, how):
+        fill_value = np.nan
+
+        dense_evens = pd.DataFrame({'A': list(range(0, 200, 2)),
+                                    'B': np.random.randint(0, 100, size=100)})
+
+        dense_threes = pd.DataFrame({'A': list(range(0, 300, 3)),
+                                     'B': np.random.randint(0, 100, size=100)})
+
         dense_merge = dense_evens.merge(dense_threes, how=how, on='A')
 
         # If you merge two dense frames together it tends to default to
@@ -1826,15 +1879,12 @@ def test_merge_two_sparse_frames(self, fill_value, how):
                                                        errors='ignore')
 
         sparse_evens = dense_evens.to_sparse(fill_value=fill_value)
-        sparse_threes = dense_threes.to_sparse(fill_value=fill_value)
+        # sparse_threes = dense_threes.to_sparse(fill_value=fill_value)
 
-        sparse_merge = sparse_evens.merge(sparse_threes, how=how, on='A')
+        sparse_merge = sparse_evens.merge(dense_threes, how=how, on='A')
 
-        assert sparse_merge.default_fill_value is fill_value
+        tm.assert_almost_equal(sparse_merge.default_fill_value, fill_value)
 
         tm.assert_sp_frame_equal(dense_merge.to_sparse(fill_value=fill_value),
                                  sparse_merge, exact_indices=False,
                                  check_dtype=False)
-
-    def test_merge_dense_sparse_frames(self, fill_value=1, how=None):
-        "pass"

pandas/tests/sparse/test_combine_concat.py

@@ -168,22 +168,18 @@ def test_concat(self):
 
         res = pd.concat([sparse, sparse])
         exp = pd.concat([self.dense1, self.dense1]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         res = pd.concat([sparse2, sparse2])
         exp = pd.concat([self.dense2, self.dense2]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         res = pd.concat([sparse, sparse2])
         exp = pd.concat([self.dense1, self.dense2]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         res = pd.concat([sparse2, sparse])
         exp = pd.concat([self.dense2, self.dense1]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
     def test_concat_different_fill_value(self):
@@ -197,7 +193,6 @@ def test_concat_different_fill_value(self):
 
         res = pd.concat([sparse2, sparse])
         exp = pd.concat([self.dense2, self.dense1]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
     def test_concat_different_columns(self):
@@ -220,12 +215,10 @@ def test_concat_different_columns(self):
 
         res = pd.concat([sparse, sparse3])
         exp = pd.concat([self.dense1, self.dense3]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         res = pd.concat([sparse3, sparse])
         exp = pd.concat([self.dense3, self.dense1]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         # different fill values
@@ -264,13 +257,11 @@ def test_concat_series(self):
             res = pd.concat([sparse, sparse2[col]])
             exp = pd.concat([self.dense1,
                              self.dense2[col]]).to_sparse(fill_value=0)
-            exp._default_fill_value = np.nan
             tm.assert_sp_frame_equal(res, exp)
 
             res = pd.concat([sparse2[col], sparse])
             exp = pd.concat([self.dense2[col],
                              self.dense1]).to_sparse(fill_value=0)
-            exp._default_fill_value = np.nan
             tm.assert_sp_frame_equal(res, exp)
 
     def test_concat_axis1(self):
@@ -294,13 +285,13 @@ def test_concat_axis1(self):
         res = pd.concat([sparse, sparse3], axis=1)
         exp = pd.concat([self.dense1, self.dense3],
                         axis=1).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        # exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         res = pd.concat([sparse3, sparse], axis=1)
         exp = pd.concat([self.dense3, self.dense1],
                         axis=1).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        # exp._default_fill_value = np.nan
         tm.assert_sp_frame_equal(res, exp)
 
         # different fill values

pandas/tests/sparse/test_indexing.py

@@ -615,32 +615,29 @@ def test_getitem(self):
         tm.assert_sp_frame_equal(sparse.iloc[[1, 2]],
                                  orig.iloc[[1, 2]].to_sparse())
 
-    def test_getitem_fill_value(self):
+    @pytest.mark.parametrize('fill_value', [0, 1, np.nan, None])
+    def test_getitem_fill_value(self, fill_value):
         orig = pd.DataFrame([[1, np.nan, 0],
                              [2, 3, np.nan],
                              [0, np.nan, 4],
                              [0, np.nan, 5]],
                             columns=list('xyz'))
-        sparse = orig.to_sparse(fill_value=0)
+        sparse = orig.to_sparse(fill_value=fill_value)
 
         tm.assert_sp_series_equal(sparse['y'],
                                   orig['y'].to_sparse(fill_value=0))
 
-        exp = orig[['x']].to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig[['x']].to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse[['x']], exp)
 
-        exp = orig[['z', 'x']].to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig[['z', 'x']].to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse[['z', 'x']], exp)
 
         indexer = [True, False, True, True]
-        exp = orig[indexer].to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig[indexer].to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse[indexer], exp)
 
-        exp = orig.iloc[[1, 2]].to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig.iloc[[1, 2]].to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse.iloc[[1, 2]], exp)
 
     def test_loc(self):
@@ -877,24 +874,22 @@ def test_take(self):
         tm.assert_sp_frame_equal(sparse.take([-1, -2]),
                                  orig.take([-1, -2]).to_sparse())
 
-    def test_take_fill_value(self):
+    @pytest.mark.parametrize('fill_value', [0, 1, np.nan, None])
+    def test_take_fill_value(self, fill_value):
         orig = pd.DataFrame([[1, np.nan, 0],
                              [2, 3, np.nan],
                              [0, np.nan, 4],
                              [0, np.nan, 5]],
                             columns=list('xyz'))
-        sparse = orig.to_sparse(fill_value=0)
+        sparse = orig.to_sparse(fill_value=fill_value)
 
-        exp = orig.take([0]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig.take([0]).to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse.take([0]), exp)
 
-        exp = orig.take([0, 1]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig.take([0, 1]).to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse.take([0, 1]), exp)
 
-        exp = orig.take([-1, -2]).to_sparse(fill_value=0)
-        exp._default_fill_value = np.nan
+        exp = orig.take([-1, -2]).to_sparse(fill_value=fill_value)
         tm.assert_sp_frame_equal(sparse.take([-1, -2]), exp)
 
     def test_reindex(self):