Squashed version of the commits below.

This is an implementation of quick shift logic

Added a vbench to reflect quick shift implementation

This change is a working version that gives the performance improvement and passes tests. Refine in next steps.

Slightly modified and cleaner logic.

Removed unused indexer, _shift_indexer

Fixed the failing tests for SparseDataFrame

Author: gouthambs

pandas/core/common.py

@@ -2003,18 +2003,6 @@ def intersection(*seqs):
     return type(seqs[0])(list(result))
 
 
-def _shift_indexer(N, periods):
-    # small reusable utility
-    indexer = np.zeros(N, dtype=int)
-
-    if periods > 0:
-        indexer[periods:] = np.arange(N - periods)
-    else:
-        indexer[:periods] = np.arange(-periods, N)
-
-    return indexer
-
-
 def _asarray_tuplesafe(values, dtype=None):
     from pandas.core.index import Index
 

pandas/core/generic.py

@@ -3197,9 +3197,7 @@ def shift(self, periods=1, freq=None, axis=0, **kwds):
             return self
 
         if freq is None and not len(kwds):
-            block_axis = self._get_block_manager_axis(axis)
-            indexer = com._shift_indexer(len(self._get_axis(axis)), periods)
-            new_data = self._data.shift(indexer=indexer, periods=periods, axis=block_axis)
+            new_data = self._data.shift(periods=periods, axis=axis)
         else:
             return self.tshift(periods, freq, **kwds)
 

pandas/core/internals.py

@@ -956,13 +956,12 @@ def diff(self, n):
         return [make_block(new_values, self.items, self.ref_items,
                            ndim=self.ndim, fastpath=True)]
 
-    def shift(self, indexer, periods, axis=0):
+    def shift(self, periods, axis=0):
         """ shift the block by periods, possibly upcast """
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
         new_values, fill_value = com._maybe_upcast(self.values)
-        new_values = np.roll(self.values.T,periods,axis=axis)
-
+        new_values = np.roll(new_values.T,periods,axis=axis)
         axis_indexer = [ slice(None) ] * self.ndim
         if periods > 0:
             axis_indexer[axis] = slice(None,periods)
@@ -972,7 +971,7 @@ def shift(self, indexer, periods, axis=0):
 
         return [make_block(new_values.T, self.items, self.ref_items,
                            ndim=self.ndim, fastpath=True)]
-
+        
     def eval(self, func, other, raise_on_error=True, try_cast=False):
         """
         evaluate the block; return result block from the result
@@ -1894,9 +1893,20 @@ def fillna(self, value, inplace=False, downcast=None):
         values = self.values if inplace else self.values.copy()
         return [self.make_block(values.get_values(value), fill_value=value)]
 
-    def shift(self, indexer, periods, axis=0):
+    @classmethod
+    def _shift_indexer(cls,N, periods):
+        # small reusable utility
+        indexer = np.zeros(N, dtype=int)
+        
+        if periods > 0:
+            indexer[periods:] = np.arange(N - periods)
+        else:
+            indexer[:periods] = np.arange(-periods, N)
+        return indexer
+        
+    def shift(self, periods, axis=0):
         """ shift the block by periods """
-
+        indexer = self._shift_indexer(len(self.values.T),periods)
         new_values = self.values.to_dense().take(indexer)
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also