ENH: first cut at SparseList data structure, #436

Author: wesm

pandas/sparse/api.py

@@ -1,4 +1,5 @@
 from pandas.sparse.array import SparseArray
+from pandas.sparse.list import SparseList
 from pandas.sparse.series import SparseSeries
 from pandas.sparse.frame import SparseDataFrame
 from pandas.sparse.panel import SparsePanel

pandas/sparse/array.py

@@ -105,7 +105,7 @@ class SparseArray(np.ndarray):
     sp_index = None
     fill_value = None
 
-    def __new__(cls, data, sparse_index=None, kind='block', fill_value=None,
+    def __new__(cls, data, sparse_index=None, kind='integer', fill_value=None,
                 copy=False):
 
         is_sparse_array = isinstance(data, SparseArray)

pandas/sparse/list.py

@@ -0,0 +1,126 @@
+import numpy as np
+
+from pandas.sparse.array import SparseArray
+import pandas._sparse as splib
+
+class SparseList(object):
+    """
+    Data structure for accumulating data to be converted into a
+    SparseArray. Has similar API to the standard Python list
+    """
+
+    def __init__(self, data=None, fill_value=np.nan):
+        self.fill_value = fill_value
+        self._chunks = []
+
+        if data is not None:
+            self.append(data)
+
+    def __repr__(self):
+        contents = '\n'.join(repr(c) for c in self._chunks)
+        return '%s\n%s' % (object.__repr__(self), contents)
+
+    def __len__(self):
+        return sum(len(c) for c in self._chunks)
+
+    def __getitem__(self, i):
+        if i < 0:
+            if i + len(self) < 0:  # pragma: no cover
+                raise ValueError('%d out of range' % i)
+            i += len(self)
+
+        passed = 0
+        j = 0
+        while i >= passed + len(self._chunks[j]):
+            passed += len(self._chunks[j])
+            j += 1
+        return self._chunks[j][i - passed]
+
+    def __setitem__(self, i, value):
+        raise NotImplementedError
+
+    @property
+    def nchunks(self):
+        return len(self._chunks)
+
+    @property
+    def is_consolidated(self):
+        return self.nchunks == 1
+
+    def consolidate(self, inplace=True):
+        """
+        Internally consolidate chunks of data
+
+        Parameters
+        ----------
+        inplace : boolean, default True
+            Modify the calling object instead of constructing a new one
+
+        Returns
+        -------
+        splist : SparseList
+            If inplace=False, new object, otherwise reference to existing
+            object
+        """
+        if not inplace:
+            result = self.copy()
+        else:
+            result = self
+
+        if result.is_consolidated:
+            return result
+
+        result._consolidate_inplace()
+        return result
+
+    def _consolidate_inplace(self):
+        new_values = np.concatenate([c.sp_values for c in self._chunks])
+        new_index = _concat_sparse_indexes([c.sp_index for c in self._chunks])
+        new_arr = SparseArray(new_values, sparse_index=new_index,
+                              fill_value=self.fill_value)
+        self._chunks = [new_arr]
+
+    def copy(self):
+        """
+        Return copy of the list
+
+        Returns
+        -------
+        new_list : SparseList
+        """
+        new_splist = SparseList(fill_value=self.fill_value)
+        new_splist._chunks = list(self._chunks)
+        return new_splist
+
+    def to_array(self):
+        """
+        Return SparseArray from data stored in the SparseList
+
+        Returns
+        -------
+        sparr : SparseArray
+        """
+        self.consolidate(inplace=True)
+        return self._chunks[0]
+
+    def append(self, value):
+        if np.isscalar(value):
+            value = [value]
+
+        sparr = SparseArray(value, fill_value=self.fill_value)
+        self._chunks.append(sparr)
+        self._consolidated = False
+
+
+def _concat_sparse_indexes(indexes):
+    all_indices = []
+    total_length = 0
+
+    for index in indexes:
+        # increment by offset
+        inds = index.to_int_index().indices + total_length
+
+        all_indices.append(inds)
+        total_length += index.length
+
+    return splib.IntIndex(total_length, np.concatenate(all_indices))

pandas/sparse/tests/test_list.py

@@ -0,0 +1,82 @@
+import unittest
+
+from numpy import nan
+import numpy as np
+
+from pandas.sparse.api import SparseList, SparseArray
+from pandas.util.testing import assert_almost_equal
+
+from test_sparse import assert_sp_array_equal
+
+
+class TestSparseList(unittest.TestCase):
+
+    def setUp(self):
+        self.na_data = np.array([nan, nan, 1, 2, 3, nan, 4, 5, nan, 6])
+        self.zero_data = np.array([0, 0, 1, 2, 3, 0, 4, 5, 0, 6])
+
+    def test_append_na(self):
+        arr = self.na_data
+        splist = SparseList()
+        splist.append(arr[:5])
+        splist.append(arr[5])
+        splist.append(arr[6:])
+
+        sparr = splist.to_array()
+        assert_sp_array_equal(sparr, SparseArray(arr))
+
+    def test_append_zero(self):
+        arr = self.zero_data
+        splist = SparseList(fill_value=0)
+        splist.append(arr[:5])
+        splist.append(arr[5])
+        splist.append(arr[6:])
+
+        sparr = splist.to_array()
+        assert_sp_array_equal(sparr, SparseArray(arr, fill_value=0))
+
+    def test_consolidate(self):
+        arr = self.na_data
+        exp_sparr = SparseArray(arr)
+
+        splist = SparseList()
+        splist.append(arr[:5])
+        splist.append(arr[5])
+        splist.append(arr[6:])
+
+        consol = splist.consolidate(inplace=False)
+        self.assert_(consol.nchunks == 1)
+        self.assert_(splist.nchunks == 3)
+        assert_sp_array_equal(consol.to_array(), exp_sparr)
+
+        splist.consolidate()
+        self.assert_(splist.nchunks == 1)
+        assert_sp_array_equal(splist.to_array(), exp_sparr)
+
+    def test_copy(self):
+        arr = self.na_data
+        exp_sparr = SparseArray(arr)
+
+        splist = SparseList()
+        splist.append(arr[:5])
+        splist.append(arr[5])
+
+        cp = splist.copy()
+        cp.append(arr[6:])
+        self.assertEquals(splist.nchunks, 2)
+        assert_sp_array_equal(cp.to_array(), exp_sparr)
+
+    def test_getitem(self):
+        arr = self.na_data
+        splist = SparseList()
+        splist.append(arr[:5])
+        splist.append(arr[5])
+        splist.append(arr[6:])
+
+        for i in range(len(arr)):
+            assert_almost_equal(splist[i], arr[i])
+
+if __name__ == '__main__':
+    import nose
+    nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],
+                   exit=False)

pandas/sparse/tests/test_sparse.py

@@ -60,12 +60,17 @@ def _test_data2_zero():
     return arr, index
 
 def assert_sp_series_equal(a, b):
-    assert_equal(a.sp_values, b.sp_values)
-    assert(a.sp_index.equals(b.sp_index))
-    if np.isnan(a.fill_value):
-        assert(np.isnan(b.fill_value))
+    assert(a.index.equals(b.index))
+    assert_sp_array_equal(a, b)
+
+def assert_sp_array_equal(left, right):
+    assert_almost_equal(left.sp_values, right.sp_values)
+    assert(left.sp_index.equals(right.sp_index))
+    if np.isnan(left.fill_value):
+        assert(np.isnan(right.fill_value))
     else:
-        assert(a.fill_value == b.fill_value)
+        assert(left.fill_value == right.fill_value)
+
 
 def assert_sp_frame_equal(left, right, exact_indices=True):
     """