TST: unit tests, and custom delete for Index objects

Author: wesm

RELEASE.rst

@@ -97,6 +97,9 @@ Release notes
   * Note that under ther hood there is a new essentially "lazy evaluation"
     scheme within respect to adding columns to DataFrame. During some
     operations, like-typed blocks will be "consolidated" but not before.
+* `DataFrame` accessing columns repeatedly is now significantly faster than
+  `DataMatrix` used to be in 0.3.0 due to an internal Series caching mechanism
+  (which are all views on the underlying data)
 * Column ordering for mixed type data is now completely consistent in
   `DataFrame`. In prior releases, there was inconsistent column ordering in
   `DataMatrix`

pandas/core/index.py

@@ -296,6 +296,9 @@ def slice_locs(self, start=None, end=None):
 
         return beg_slice, end_slice
 
+    def delete(self, loc):
+        arr = np.delete(np.asarray(self), loc)
+        return Index(arr)
 
 class DateIndex(Index):
     pass
@@ -768,6 +771,10 @@ def _assert_can_do_setop(self, other):
 
         assert(self.nlevels == other.nlevels)
 
+    def delete(self, loc):
+        new_labels = [np.delete(lab, loc) for lab in self.labels]
+        return MultiIndex(levels=self.levels, labels=new_labels)
+
     get_major_bounds = slice_locs
 
     __bounds = None

pandas/core/indexing.py

@@ -105,7 +105,7 @@ def _getitem_tuple(self, key):
         if isinstance(self.frame.index, MultiIndex):
             try:
                 return self.frame.xs(key)
-            except KeyError:
+            except (KeyError, TypeError):
                 # could do something more intelligent here? like raising the
                 # exception if each tuple value are in the levels?
                 pass

pandas/core/series.py

@@ -264,10 +264,14 @@ def __getitem__(self, key):
         try:
             if isinstance(self.index, MultiIndex):
                 return self._multilevel_index(key)
-            elif isinstance(key, int):
-                return self._regular_index(key)
             else:
-                return self._regular_index(key)
+                values = self.values
+                try:
+                    return values[self.index.get_loc(key)]
+                except KeyError:
+                    if isinstance(key, (int, np.integer)):
+                        return values[key]
+                    raise Exception('Requested index not in this series!')
         except Exception:
             pass
 
@@ -293,16 +297,6 @@ def _index_with(indexer):
             key = np.asarray(key)
             return _index_with(key)
 
-    def _regular_index(self, key):
-        values = self.values
-
-        try:
-            return values[self.index.get_loc(key)]
-        except KeyError:
-            if isinstance(key, (int, np.integer)):
-                return values[key]
-            raise Exception('Requested index not in this series!')
-
     def _multilevel_index(self, key):
         values = self.values
         try:
@@ -720,7 +714,11 @@ def values(self):
         -------
         arr : numpy.ndarray
         """
-        return self.view(ndarray)
+        try:
+            return self._values
+        except AttributeError:
+            self._values = self.view(ndarray)
+            return self._values
 
     def iteritems(self):
         """

pandas/tests/test_frame.py

@@ -1501,6 +1501,51 @@ def test_dropIncompleteRows(self):
         samesize_frame = frame.dropna(subset=['bar'])
         self.assert_(samesize_frame.index.equals(self.frame.index))
 
+    def test_dropna(self):
+        df = DataFrame(np.random.randn(6, 4))
+        df[2][:2] = nan
+
+        dropped = df.dropna(axis=1)
+        expected = df.ix[:, [0, 1, 3]]
+        assert_frame_equal(dropped, expected)
+
+        dropped = df.dropna(axis=0)
+        expected = df.ix[range(2, 6)]
+        assert_frame_equal(dropped, expected)
+
+        # threshold
+        dropped = df.dropna(axis=1, thresh=5)
+        expected = df.ix[:, [0, 1, 3]]
+        assert_frame_equal(dropped, expected)
+
+        dropped = df.dropna(axis=0, thresh=4)
+        expected = df.ix[range(2, 6)]
+        assert_frame_equal(dropped, expected)
+
+        dropped = df.dropna(axis=1, thresh=4)
+        assert_frame_equal(dropped, df)
+
+        dropped = df.dropna(axis=1, thresh=3)
+        assert_frame_equal(dropped, df)
+
+        # subset
+        dropped = df.dropna(axis=0, subset=[0, 1, 3])
+        assert_frame_equal(dropped, df)
+
+        # all
+        dropped = df.dropna(axis=1, how='all')
+        assert_frame_equal(dropped, df)
+
+        df[2] = nan
+        dropped = df.dropna(axis=1, how='all')
+        expected = df.ix[:, [0, 1, 3]]
+        assert_frame_equal(dropped, expected)
+
+    def test_dropna_corner(self):
+        # bad input
+        self.assertRaises(ValueError, self.frame.dropna, how='foo')
+        self.assertRaises(ValueError, self.frame.dropna, how=None)
+
     def test_fillna(self):
         self.tsframe['A'][:5] = nan
         self.tsframe['A'][-5:] = nan

pandas/tests/test_multilevel.py

@@ -35,10 +35,35 @@ def setUp(self):
         self.frame = DataFrame(np.random.randn(10, 3), index=index,
                                columns=['A', 'B', 'C'])
 
+        tm.N = 100
         self.tdf = tm.makeTimeDataFrame()
         self.ymd = self.tdf.groupby([lambda x: x.year, lambda x: x.month,
                                      lambda x: x.day]).sum()
 
+    def test_pickle(self):
+        import cPickle
+        def _test_roundtrip(frame):
+            pickled = cPickle.dumps(frame)
+            unpickled = cPickle.loads(pickled)
+            assert_frame_equal(frame, unpickled)
+
+        _test_roundtrip(self.frame)
+        _test_roundtrip(self.frame.T)
+        _test_roundtrip(self.ymd)
+        _test_roundtrip(self.ymd.T)
+
+    def test_repr_to_string(self):
+        repr(self.frame)
+        repr(self.ymd)
+        repr(self.frame.T)
+        repr(self.ymd.T)
+
+        buf = StringIO()
+        self.frame.toString(buf=buf)
+        self.ymd.toString(buf=buf)
+        self.frame.T.toString(buf=buf)
+        self.ymd.T.toString(buf=buf)
+
     def test_getitem_simple(self):
         df = self.frame.T
 
@@ -80,9 +105,12 @@ def test_getitem_toplevel(self):
         assert_frame_equal(result, expected)
 
         result = df['bar']
+        result2 = df.ix[:, 'bar']
+
         expected = df.reindex(columns=df.columns[3:5])
         expected.columns = expected.columns.droplevel(0)
         assert_frame_equal(result, expected)
+        assert_frame_equal(result, result2)
 
     def test_getitem_partial(self):
         ymd = self.ymd.T
@@ -92,11 +120,47 @@ def test_getitem_partial(self):
         expected.columns = expected.columns.droplevel(0).droplevel(0)
         assert_frame_equal(result, expected)
 
+    def test_setitem_change_dtype(self):
+        dft = self.frame.T
+        s = dft['foo', 'two']
+        dft['foo', 'two'] = s > s.median()
+        assert_series_equal(dft['foo', 'two'], s > s.median())
+        self.assert_(isinstance(dft._data.blocks[1].items, MultiIndex))
+
+        reindexed = dft.reindex(columns=[('foo', 'two')])
+        assert_series_equal(reindexed['foo', 'two'], s > s.median())
+
     def test_fancy_slice_partial(self):
-        pass
+        result = self.frame.ix['bar':'baz']
+        expected = self.frame[3:7]
+        assert_frame_equal(result, expected)
 
-    def test_fancy_select_toplevel(self):
-        pass
+        result = self.ymd.ix[(2000,2):(2000,4)]
+        lev = self.ymd.index.labels[1]
+        expected = self.ymd[(lev >= 1) & (lev <= 3)]
+        assert_frame_equal(result, expected)
+
+    def test_sortlevel(self):
+        df = self.frame.copy()
+        df.index = np.arange(len(df))
+        self.assertRaises(Exception, df.sortlevel, 0)
+
+        # axis=1
+
+    def test_sortlevel_mixed(self):
+        sorted_before = self.frame.sortlevel(1)
+
+        df = self.frame.copy()
+        df['foo'] = 'bar'
+        sorted_after = df.sortlevel(1)
+        assert_frame_equal(sorted_before, sorted_after.drop(['foo'], axis=1))
+
+        dft = self.frame.T
+        sorted_before = dft.sortlevel(1, axis=1)
+        dft['foo', 'three'] = 'bar'
+
+        sorted_after = dft.sortlevel(1, axis=1)
+        assert_frame_equal(sorted_before, sorted_after.drop(['foo'], axis=1))
 
     def test_alignment(self):
         pass