From 695fcd5a08bf599d6bbe9e70649f5e114273676e Mon Sep 17 00:00:00 2001 From: Dale Jung Date: Tue, 17 Sep 2013 03:55:40 -0400 Subject: [PATCH 1/2] ENH: Add makePeriodPanel --- pandas/util/testing.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 0718dc8926011..0481a522dabe8 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -460,12 +460,12 @@ def makeTimeDataFrame(nper=None): return DataFrame(data) -def getPeriodData(): - return dict((c, makePeriodSeries()) for c in getCols(K)) +def getPeriodData(nper=None): + return dict((c, makePeriodSeries(nper)) for c in getCols(K)) -def makePeriodFrame(): - data = getPeriodData() +def makePeriodFrame(nper=None): + data = getPeriodData(nper) return DataFrame(data) @@ -474,6 +474,10 @@ def makePanel(nper=None): data = dict((c, makeTimeDataFrame(nper)) for c in cols) return Panel.fromDict(data) +def makePeriodPanel(nper=None): + cols = ['Item' + c for c in string.ascii_uppercase[:K - 1]] + data = dict((c, makePeriodFrame(nper)) for c in cols) + return Panel.fromDict(data) def makePanel4D(nper=None): return Panel4D(dict(l1=makePanel(nper), l2=makePanel(nper), From 00827fbc0e3e19c85d97ad173a0351cec5b3b908 Mon Sep 17 00:00:00 2001 From: Dale Jung Date: Tue, 17 Sep 2013 05:05:38 -0400 Subject: [PATCH 2/2] BUG: (GH4853) Fixed Panel.shift/tshift to use freq. TST: Added Panel.tshift test case RST: Added to release notes CLN: on top of dale-jung 4863, remove shift from series.py, move to generic.py --- doc/source/release.rst | 1 + pandas/core/datetools.py | 20 ++++++++++ pandas/core/frame.py | 49 ----------------------- pandas/core/generic.py | 73 +++++++++++++++++++++++++++++++--- pandas/core/internals.py | 22 ++++++++--- pandas/core/panel.py | 8 +++- pandas/core/series.py | 78 ------------------------------------- pandas/sparse/series.py | 2 +- pandas/tests/test_panel.py | 38 ++++++++++++++++++ pandas/tests/test_series.py | 2 - 10 files changed, 151 insertions(+), 142 deletions(-) diff --git a/doc/source/release.rst b/doc/source/release.rst index 793d52223b6f5..d747505593c94 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -428,6 +428,7 @@ Bug Fixes single column and passing a list for ``ascending``, the argument for ``ascending`` was being interpreted as ``True`` (:issue:`4839`, :issue:`4846`) + - Fixed ``Panel.tshift`` not working. Added `freq` support to ``Panel.shift`` (:issue:`4853`) pandas 0.12.0 ------------- diff --git a/pandas/core/datetools.py b/pandas/core/datetools.py index 228dc7574f8f3..91a29259d8f2f 100644 --- a/pandas/core/datetools.py +++ b/pandas/core/datetools.py @@ -35,3 +35,23 @@ isBusinessDay = BDay().onOffset isMonthEnd = MonthEnd().onOffset isBMonthEnd = BMonthEnd().onOffset + +def _resolve_offset(freq, kwds): + if 'timeRule' in kwds or 'offset' in kwds: + offset = kwds.get('offset', None) + offset = kwds.get('timeRule', offset) + if isinstance(offset, compat.string_types): + offset = getOffset(offset) + warn = True + else: + offset = freq + warn = False + + if warn: + import warnings + warnings.warn("'timeRule' and 'offset' parameters are deprecated," + " please use 'freq' instead", + FutureWarning) + + return offset + diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 78ef806a45dcb..70fcc2c9d9c0a 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -3497,55 +3497,6 @@ def diff(self, periods=1): new_data = self._data.diff(periods) return self._constructor(new_data) - def shift(self, periods=1, freq=None, **kwds): - """ - Shift the index of the DataFrame by desired number of periods with an - optional time freq - - Parameters - ---------- - periods : int - Number of periods to move, can be positive or negative - freq : DateOffset, timedelta, or time rule string, optional - Increment to use from datetools module or time rule (e.g. 'EOM') - - Notes - ----- - If freq is specified then the index values are shifted but the data - if not realigned - - Returns - ------- - shifted : DataFrame - """ - from pandas.core.series import _resolve_offset - - if periods == 0: - return self - - offset = _resolve_offset(freq, kwds) - - if isinstance(offset, compat.string_types): - offset = datetools.to_offset(offset) - - if offset is None: - indexer = com._shift_indexer(len(self), periods) - new_data = self._data.shift(indexer, periods) - elif isinstance(self.index, PeriodIndex): - orig_offset = datetools.to_offset(self.index.freq) - if offset == orig_offset: - new_data = self._data.copy() - new_data.axes[1] = self.index.shift(periods) - else: - msg = ('Given freq %s does not match PeriodIndex freq %s' % - (offset.rule_code, orig_offset.rule_code)) - raise ValueError(msg) - else: - new_data = self._data.copy() - new_data.axes[1] = self.index.shift(periods, offset) - - return self._constructor(new_data) - #---------------------------------------------------------------------- # Function application diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 2f6bc13983f93..53d3687854cac 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -11,8 +11,10 @@ import pandas.core.indexing as indexing from pandas.core.indexing import _maybe_convert_indices from pandas.tseries.index import DatetimeIndex +from pandas.tseries.period import PeriodIndex from pandas.core.internals import BlockManager import pandas.core.common as com +import pandas.core.datetools as datetools from pandas import compat, _np_version_under1p7 from pandas.compat import map, zip, lrange from pandas.core.common import (isnull, notnull, is_list_like, @@ -2667,7 +2669,40 @@ def cummin(self, axis=None, skipna=True): result = np.minimum.accumulate(y, axis) return self._wrap_array(result, self.axes, copy=False) - def tshift(self, periods=1, freq=None, **kwds): + def shift(self, periods=1, freq=None, axis=0, **kwds): + """ + Shift the index of the DataFrame by desired number of periods with an + optional time freq + + Parameters + ---------- + periods : int + Number of periods to move, can be positive or negative + freq : DateOffset, timedelta, or time rule string, optional + Increment to use from datetools module or time rule (e.g. 'EOM') + + Notes + ----- + If freq is specified then the index values are shifted but the data + if not realigned + + Returns + ------- + shifted : DataFrame + """ + if periods == 0: + return self + + if freq is None and not len(kwds): + block_axis = self._get_block_manager_axis(axis) + indexer = com._shift_indexer(len(self), periods) + new_data = self._data.shift(indexer, periods, axis=block_axis) + else: + return self.tshift(periods, freq, **kwds) + + return self._constructor(new_data) + + def tshift(self, periods=1, freq=None, axis=0, **kwds): """ Shift the time index, using the index's frequency if available @@ -2677,6 +2712,8 @@ def tshift(self, periods=1, freq=None, **kwds): Number of periods to move, can be positive or negative freq : DateOffset, timedelta, or time rule string, default None Increment to use from datetools module or time rule (e.g. 'EOM') + axis : int or basestring + Corresponds to the axis that contains the Index Notes ----- @@ -2686,19 +2723,45 @@ def tshift(self, periods=1, freq=None, **kwds): Returns ------- - shifted : Series + shifted : NDFrame """ + from pandas.core.datetools import _resolve_offset + + index = self._get_axis(axis) if freq is None: - freq = getattr(self.index, 'freq', None) + freq = getattr(index, 'freq', None) if freq is None: - freq = getattr(self.index, 'inferred_freq', None) + freq = getattr(index, 'inferred_freq', None) if freq is None: msg = 'Freq was not given and was not set in the index' raise ValueError(msg) - return self.shift(periods, freq, **kwds) + + if periods == 0: + return self + + offset = _resolve_offset(freq, kwds) + + if isinstance(offset, compat.string_types): + offset = datetools.to_offset(offset) + + block_axis = self._get_block_manager_axis(axis) + if isinstance(index, PeriodIndex): + orig_offset = datetools.to_offset(index.freq) + if offset == orig_offset: + new_data = self._data.copy() + new_data.axes[block_axis] = index.shift(periods) + else: + msg = ('Given freq %s does not match PeriodIndex freq %s' % + (offset.rule_code, orig_offset.rule_code)) + raise ValueError(msg) + else: + new_data = self._data.copy() + new_data.axes[block_axis] = index.shift(periods, offset) + + return self._constructor(new_data) def truncate(self, before=None, after=None, copy=True): """Function truncate a sorted DataFrame / Series before and/or after diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 11ce27b078b18..4b9fdb0422526 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -758,17 +758,27 @@ def diff(self, n): new_values = com.diff(self.values, n, axis=1) return [make_block(new_values, self.items, self.ref_items, ndim=self.ndim, fastpath=True)] - def shift(self, indexer, periods): + def shift(self, indexer, periods, axis=0): """ shift the block by periods, possibly upcast """ - new_values = self.values.take(indexer, axis=1) + new_values = self.values.take(indexer, axis=axis) # convert integer to float if necessary. need to do a lot more than # that, handle boolean etc also new_values, fill_value = com._maybe_upcast(new_values) - if periods > 0: - new_values[:, :periods] = fill_value + + # 1-d + if self.ndim == 1: + if periods > 0: + new_values[:periods] = fill_value + else: + new_values[periods:] = fill_value + + # 2-d else: - new_values[:, periods:] = fill_value + if periods > 0: + new_values[:, :periods] = fill_value + else: + new_values[:, periods:] = fill_value return [make_block(new_values, self.items, self.ref_items, ndim=self.ndim, fastpath=True)] def eval(self, func, other, raise_on_error=True, try_cast=False): @@ -1547,7 +1557,7 @@ def fillna(self, value, inplace=False, downcast=None): values = self.values if inplace else self.values.copy() return [ self.make_block(values.get_values(value), fill_value=value) ] - def shift(self, indexer, periods): + def shift(self, indexer, periods, axis=0): """ shift the block by periods """ new_values = self.values.to_dense().take(indexer) diff --git a/pandas/core/panel.py b/pandas/core/panel.py index 6f02b49326e4d..45101b1e2afd5 100644 --- a/pandas/core/panel.py +++ b/pandas/core/panel.py @@ -1017,7 +1017,7 @@ def count(self, axis='major'): return self._wrap_result(result, axis) - def shift(self, lags, axis='major'): + def shift(self, lags, freq=None, axis='major'): """ Shift major or minor axis by specified number of leads/lags. Drops periods right now compared with DataFrame.shift @@ -1036,6 +1036,9 @@ def shift(self, lags, axis='major'): major_axis = self.major_axis minor_axis = self.minor_axis + if freq: + return self.tshift(lags, freq, axis=axis) + if lags > 0: vslicer = slice(None, -lags) islicer = slice(lags, None) @@ -1058,6 +1061,9 @@ def shift(self, lags, axis='major'): return self._constructor(values, items=items, major_axis=major_axis, minor_axis=minor_axis) + def tshift(self, periods=1, freq=None, axis='major', **kwds): + return super(Panel, self).tshift(periods, freq, axis, **kwds) + def truncate(self, before=None, after=None, axis='major'): """Function truncates a sorted Panel before and/or after some particular values on the requested axis diff --git a/pandas/core/series.py b/pandas/core/series.py index beb398dfe6fd0..9f7ab0cb0346b 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -59,8 +59,6 @@ _np_version_under1p6 = LooseVersion(_np_version) < '1.6' _np_version_under1p7 = LooseVersion(_np_version) < '1.7' -_SHOW_WARNINGS = True - class _TimeOp(object): """ Wrapper around Series datetime/time/timedelta arithmetic operations. @@ -2917,62 +2915,6 @@ def last_valid_index(self): #---------------------------------------------------------------------- # Time series-oriented methods - def shift(self, periods=1, freq=None, copy=True, **kwds): - """ - Shift the index of the Series by desired number of periods with an - optional time offset - - Parameters - ---------- - periods : int - Number of periods to move, can be positive or negative - freq : DateOffset, timedelta, or offset alias string, optional - Increment to use from datetools module or time rule (e.g. 'EOM') - - Returns - ------- - shifted : Series - """ - if periods == 0: - return self.copy() - - offset = _resolve_offset(freq, kwds) - - if isinstance(offset, compat.string_types): - offset = datetools.to_offset(offset) - - def _get_values(): - values = self.values - if copy: - values = values.copy() - return values - - if offset is None: - dtype, fill_value = _maybe_promote(self.dtype) - new_values = pa.empty(len(self), dtype=dtype) - - if periods > 0: - new_values[periods:] = self.values[:-periods] - new_values[:periods] = fill_value - elif periods < 0: - new_values[:periods] = self.values[-periods:] - new_values[periods:] = fill_value - - return self._constructor(new_values, index=self.index, name=self.name) - elif isinstance(self.index, PeriodIndex): - orig_offset = datetools.to_offset(self.index.freq) - if orig_offset == offset: - return self._constructor( - _get_values(), self.index.shift(periods), - name=self.name) - msg = ('Given freq %s does not match PeriodIndex freq %s' % - (offset.rule_code, orig_offset.rule_code)) - raise ValueError(msg) - else: - return self._constructor(_get_values(), - index=self.index.shift(periods, offset), - name=self.name) - def asof(self, where): """ Return last good (non-NaN) value in TimeSeries if value is NaN for @@ -3317,26 +3259,6 @@ def _try_cast(arr, take_fast_path): return subarr -def _resolve_offset(freq, kwds): - if 'timeRule' in kwds or 'offset' in kwds: - offset = kwds.get('offset', None) - offset = kwds.get('timeRule', offset) - if isinstance(offset, compat.string_types): - offset = datetools.getOffset(offset) - warn = True - else: - offset = freq - warn = False - - if warn and _SHOW_WARNINGS: # pragma: no cover - import warnings - warnings.warn("'timeRule' and 'offset' parameters are deprecated," - " please use 'freq' instead", - FutureWarning) - - return offset - - # backwards compatiblity TimeSeries = Series diff --git a/pandas/sparse/series.py b/pandas/sparse/series.py index 537b88db3c1f0..5cb29d717235d 100644 --- a/pandas/sparse/series.py +++ b/pandas/sparse/series.py @@ -605,7 +605,7 @@ def shift(self, periods, freq=None, **kwds): """ Analogous to Series.shift """ - from pandas.core.series import _resolve_offset + from pandas.core.datetools import _resolve_offset offset = _resolve_offset(freq, kwds) diff --git a/pandas/tests/test_panel.py b/pandas/tests/test_panel.py index fc86a78ea684b..a498cca528043 100644 --- a/pandas/tests/test_panel.py +++ b/pandas/tests/test_panel.py @@ -1323,6 +1323,44 @@ def test_shift(self): for i, f in compat.iteritems(self.panel))) assert_panel_equal(result, expected) + def test_tshift(self): + # PeriodIndex + ps = tm.makePeriodPanel() + shifted = ps.tshift(1) + unshifted = shifted.tshift(-1) + + assert_panel_equal(unshifted, ps) + + shifted2 = ps.tshift(freq='B') + assert_panel_equal(shifted, shifted2) + + shifted3 = ps.tshift(freq=bday) + assert_panel_equal(shifted, shifted3) + + assertRaisesRegexp(ValueError, 'does not match', ps.tshift, freq='M') + + # DatetimeIndex + panel = _panel + shifted = panel.tshift(1) + unshifted = shifted.tshift(-1) + + assert_panel_equal(panel, unshifted) + + shifted2 = panel.tshift(freq=panel.major_axis.freq) + assert_panel_equal(shifted, shifted2) + + inferred_ts = Panel(panel.values, + items=panel.items, + major_axis=Index(np.asarray(panel.major_axis)), + minor_axis=panel.minor_axis) + shifted = inferred_ts.tshift(1) + unshifted = shifted.tshift(-1) + assert_panel_equal(shifted, panel.tshift(1)) + assert_panel_equal(unshifted, inferred_ts) + + no_freq = panel.ix[:, [0, 5, 7], :] + self.assertRaises(ValueError, no_freq.tshift) + def test_multiindex_get(self): ind = MultiIndex.from_tuples([('a', 1), ('a', 2), ('b', 1), ('b', 2)], names=['first', 'second']) diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py index 686df18999850..b142adbd5b949 100644 --- a/pandas/tests/test_series.py +++ b/pandas/tests/test_series.py @@ -3550,13 +3550,11 @@ def test_shift(self): self.assertRaises(ValueError, ps.shift, freq='D') # legacy support - smod._SHOW_WARNINGS = False shifted4 = ps.shift(1, timeRule='B') assert_series_equal(shifted2, shifted4) shifted5 = ps.shift(1, offset=datetools.bday) assert_series_equal(shifted5, shifted4) - smod._SHOW_WARNINGS = True def test_tshift(self): # PeriodIndex