Skip to content

ENH: Add milliseconds field support for pd.DateOffset #43598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Mar 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
62918ce
initial
isVoid Sep 16, 2021
a04bf64
docstrings
isVoid Sep 16, 2021
b61c14d
Remove unwanted change
isVoid Sep 16, 2021
c067d24
Add tests
isVoid Sep 28, 2021
d0814a8
add license
isVoid Sep 28, 2021
8e96489
Add to whats new section
isVoid Sep 28, 2021
a02be79
restructure TestDateOffset and extend with more coverage over parameters
isVoid Oct 5, 2021
255fe23
remove license
isVoid Oct 5, 2021
b5a42e6
remove stale comments
isVoid Oct 5, 2021
a5dc38e
Merge branch 'master' into add_dateoffset_milliseconds
isVoid Oct 5, 2021
073c319
Merge branch 'master' into add_dateoffset_milliseconds
jreback Oct 16, 2021
4801de3
Removing existing mul tests
isVoid Oct 20, 2021
dbadf9f
change test parameter name
isVoid Oct 20, 2021
6a98391
Merge branch 'add_dateoffset_milliseconds' of https://github.com/isVo…
isVoid Oct 20, 2021
30d3403
Add some docs for replace component usage
isVoid Oct 20, 2021
bafceb6
docstring and test fixes
isVoid Oct 22, 2021
3bd154c
add assertion to constructed dateoffset
isVoid Oct 29, 2021
4bb88df
Merge branch 'master' of https://github.com/pandas-dev/pandas into ad…
isVoid Nov 1, 2021
41e537f
raise when millisecond (replacement form) is specified
isVoid Nov 2, 2021
1da06d0
Merge branch 'master' into add_dateoffset_milliseconds
isVoid Nov 29, 2021
fcfce37
apply review comments
isVoid Dec 15, 2021
1c7d7d7
fix fail tests
isVoid Dec 15, 2021
99574ea
xfail instead of skip
isVoid Dec 29, 2021
b7eeb4d
Merge branch 'main' of git://github.com/pandas-dev/pandas into add_da…
isVoid Jan 17, 2022
e271461
Move notes to 1.5
isVoid Jan 17, 2022
3a85b4e
Merge branch 'main' of git://github.com/pandas-dev/pandas into add_da…
isVoid Jan 31, 2022
b334f69
Add alternative to message
isVoid Jan 31, 2022
f9bfadd
Merge branch 'main' of git://github.com/pandas-dev/pandas into add_da…
isVoid Mar 7, 2022
0fb51a1
Move replacement doc to docstring
isVoid Mar 8, 2022
40f986a
Merge branch 'main' of github.com:pandas-dev/pandas into add_dateoffs…
isVoid Mar 16, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/user_guide/timeseries.rst
Original file line number Diff line number Diff line change
Expand Up @@ -869,6 +869,7 @@ arithmetic operator (``+``) can be used to perform the shift.
friday + two_business_days
(friday + two_business_days).day_name()


Most ``DateOffsets`` have associated frequencies strings, or offset aliases, that can be passed
into ``freq`` keyword arguments. The available date offsets and associated frequency strings can be found below:

Expand Down
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.5.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ Other enhancements
- Implemented a complex-dtype :class:`Index`, passing a complex-dtype array-like to ``pd.Index`` will now retain complex dtype instead of casting to ``object`` (:issue:`45845`)
- Improved error message in :class:`~pandas.core.window.Rolling` when ``window`` is a frequency and ``NaT`` is in the rolling axis (:issue:`46087`)
- :class:`Series` and :class:`DataFrame` with ``IntegerDtype`` now supports bitwise operations (:issue:`34463`)
- Add ``milliseconds`` field support for :class:`~pandas.DateOffset` (:issue:`43371`)
-

.. ---------------------------------------------------------------------------
Expand Down
24 changes: 21 additions & 3 deletions pandas/_libs/tslibs/offsets.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -307,8 +307,9 @@ cdef _validate_business_time(t_input):

_relativedelta_kwds = {"years", "months", "weeks", "days", "year", "month",
"day", "weekday", "hour", "minute", "second",
"microsecond", "nanosecond", "nanoseconds", "hours",
"minutes", "seconds", "microseconds"}
"microsecond", "millisecond", "nanosecond",
"nanoseconds", "hours", "minutes", "seconds",
"milliseconds", "microseconds"}


cdef _determine_offset(kwds):
Expand All @@ -323,11 +324,19 @@ cdef _determine_offset(kwds):

_kwds_use_relativedelta = ('years', 'months', 'weeks', 'days',
'year', 'month', 'week', 'day', 'weekday',
'hour', 'minute', 'second', 'microsecond')
'hour', 'minute', 'second', 'microsecond',
'millisecond')

use_relativedelta = False
if len(kwds_no_nanos) > 0:
if any(k in _kwds_use_relativedelta for k in kwds_no_nanos):
if "millisecond" in kwds_no_nanos:
raise NotImplementedError(
"Using DateOffset to replace `millisecond` component in "
"datetime object is not supported. Use "
"`microsecond=timestamp.microsecond % 1000 + ms * 1000` "
"instead."
)
offset = relativedelta(**kwds_no_nanos)
use_relativedelta = True
else:
Expand Down Expand Up @@ -1223,6 +1232,9 @@ class DateOffset(RelativeDeltaOffset, metaclass=OffsetMeta):

Since 0 is a bit weird, we suggest avoiding its use.

Besides, adding a DateOffsets specified by the singular form of the date
component can be used to replace certain component of the timestamp.

Parameters
----------
n : int, default 1
Expand All @@ -1243,6 +1255,7 @@ class DateOffset(RelativeDeltaOffset, metaclass=OffsetMeta):
- hours
- minutes
- seconds
- milliseconds
- microseconds
- nanoseconds

Expand Down Expand Up @@ -1274,6 +1287,11 @@ class DateOffset(RelativeDeltaOffset, metaclass=OffsetMeta):
>>> ts = pd.Timestamp('2017-01-01 09:10:11')
>>> ts + DateOffset(months=2)
Timestamp('2017-03-01 09:10:11')
>>> ts + DateOffset(day=31)
Timestamp('2017-01-31 09:10:11')

>>> ts + pd.DateOffset(hour=8)
Timestamp('2017-01-01 08:10:11')
"""
def __setattr__(self, name, value):
raise AttributeError("DateOffset objects are immutable.")
Expand Down
152 changes: 144 additions & 8 deletions pandas/tests/tseries/offsets/test_offsets.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,18 @@

_ApplyCases = List[Tuple[BaseOffset, Dict[datetime, datetime]]]

_ARITHMETIC_DATE_OFFSET = [
"years",
"months",
"weeks",
"days",
"hours",
"minutes",
"seconds",
"milliseconds",
"microseconds",
]


class TestCommon(Base):
# executed value created by Base._get_offset
Expand Down Expand Up @@ -551,28 +563,144 @@ def test_mul(self):
assert DateOffset(2) == 2 * DateOffset(1)
assert DateOffset(2) == DateOffset(1) * 2

def test_constructor(self):

assert (self.d + DateOffset(months=2)) == datetime(2008, 3, 2)
assert (self.d - DateOffset(months=2)) == datetime(2007, 11, 2)
@pytest.mark.parametrize("relativedelta_kwd", list(liboffsets._relativedelta_kwds))
Copy link
Member

@simonjayhawkins simonjayhawkins Mar 18, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This needs to be list -> sorted to prevent pytest collection errors with pytest-xdist.

Also, for consistency with elsewhere in this module, could maybe also change "relativedelta_kwd" -> "kwd"?

def test_constructor(self, relativedelta_kwd, request):
if relativedelta_kwd == "millisecond":
request.node.add_marker(
pytest.mark.xfail(
raises=NotImplementedError,
reason="Constructing DateOffset object with `millisecond` is not "
"yet supported.",
)
)
offset = DateOffset(**{relativedelta_kwd: 2})
assert offset.kwds == {relativedelta_kwd: 2}
assert getattr(offset, relativedelta_kwd) == 2

def test_default_constructor(self):
assert (self.d + DateOffset(2)) == datetime(2008, 1, 4)

def test_is_anchored(self):
assert not DateOffset(2).is_anchored()
assert DateOffset(1).is_anchored()

d = datetime(2008, 1, 31)
assert (d + DateOffset(months=1)) == datetime(2008, 2, 29)

def test_copy(self):
assert DateOffset(months=2).copy() == DateOffset(months=2)
assert DateOffset(milliseconds=1).copy() == DateOffset(milliseconds=1)

@pytest.mark.parametrize(
"arithmatic_offset_type, expected",
zip(
_ARITHMETIC_DATE_OFFSET,
[
"2009-01-02",
"2008-02-02",
"2008-01-09",
"2008-01-03",
"2008-01-02 01:00:00",
"2008-01-02 00:01:00",
"2008-01-02 00:00:01",
"2008-01-02 00:00:00.001000000",
"2008-01-02 00:00:00.000001000",
],
),
)
def test_add(self, arithmatic_offset_type, expected):
assert DateOffset(**{arithmatic_offset_type: 1}) + self.d == Timestamp(expected)
assert self.d + DateOffset(**{arithmatic_offset_type: 1}) == Timestamp(expected)

@pytest.mark.parametrize(
"arithmatic_offset_type, expected",
zip(
_ARITHMETIC_DATE_OFFSET,
[
"2007-01-02",
"2007-12-02",
"2007-12-26",
"2008-01-01",
"2008-01-01 23:00:00",
"2008-01-01 23:59:00",
"2008-01-01 23:59:59",
"2008-01-01 23:59:59.999000000",
"2008-01-01 23:59:59.999999000",
],
),
)
def test_sub(self, arithmatic_offset_type, expected):
assert self.d - DateOffset(**{arithmatic_offset_type: 1}) == Timestamp(expected)
with pytest.raises(TypeError, match="Cannot subtract datetime from offset"):
DateOffset(**{arithmatic_offset_type: 1}) - self.d

@pytest.mark.parametrize(
"arithmatic_offset_type, n, expected",
zip(
_ARITHMETIC_DATE_OFFSET,
range(1, 10),
[
"2009-01-02",
"2008-03-02",
"2008-01-23",
"2008-01-06",
"2008-01-02 05:00:00",
"2008-01-02 00:06:00",
"2008-01-02 00:00:07",
"2008-01-02 00:00:00.008000000",
"2008-01-02 00:00:00.000009000",
],
),
)
def test_mul_add(self, arithmatic_offset_type, n, expected):
assert DateOffset(**{arithmatic_offset_type: 1}) * n + self.d == Timestamp(
expected
)
assert n * DateOffset(**{arithmatic_offset_type: 1}) + self.d == Timestamp(
expected
)
assert self.d + DateOffset(**{arithmatic_offset_type: 1}) * n == Timestamp(
expected
)
assert self.d + n * DateOffset(**{arithmatic_offset_type: 1}) == Timestamp(
expected
)

@pytest.mark.parametrize(
"arithmatic_offset_type, n, expected",
zip(
_ARITHMETIC_DATE_OFFSET,
range(1, 10),
[
"2007-01-02",
"2007-11-02",
"2007-12-12",
"2007-12-29",
"2008-01-01 19:00:00",
"2008-01-01 23:54:00",
"2008-01-01 23:59:53",
"2008-01-01 23:59:59.992000000",
"2008-01-01 23:59:59.999991000",
],
),
)
def test_mul_sub(self, arithmatic_offset_type, n, expected):
assert self.d - DateOffset(**{arithmatic_offset_type: 1}) * n == Timestamp(
expected
)
assert self.d - n * DateOffset(**{arithmatic_offset_type: 1}) == Timestamp(
expected
)

def test_leap_year(self):
d = datetime(2008, 1, 31)
assert (d + DateOffset(months=1)) == datetime(2008, 2, 29)

def test_eq(self):
offset1 = DateOffset(days=1)
offset2 = DateOffset(days=365)

assert offset1 != offset2

assert DateOffset(milliseconds=3) != DateOffset(milliseconds=7)


class TestOffsetNames:
def test_get_offset_name(self):
Expand Down Expand Up @@ -741,7 +869,15 @@ def test_month_offset_name(month_classes):


@pytest.mark.parametrize("kwd", sorted(liboffsets._relativedelta_kwds))
def test_valid_relativedelta_kwargs(kwd):
def test_valid_relativedelta_kwargs(kwd, request):
if kwd == "millisecond":
request.node.add_marker(
pytest.mark.xfail(
raises=NotImplementedError,
reason="Constructing DateOffset object with `millisecond` is not "
"yet supported.",
)
)
# Check that all the arguments specified in liboffsets._relativedelta_kwds
# are in fact valid relativedelta keyword args
DateOffset(**{kwd: 1})
Expand Down