From cdd46baea78fd5739fb72c016266f64e024edb1e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 16 Feb 2022 18:02:20 -0800 Subject: [PATCH 01/18] CI: Test on Cython 3.0 on numpydev --- ci/deps/actions-310-numpydev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 3e32665d5433f..2c53da6587425 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -15,7 +15,7 @@ dependencies: - pytz - pip - pip: - - cython==0.29.24 # GH#34014 + - cython - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" From 27a674b31e1f62a049f4ae1e8b745038948896f9 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 16 Feb 2022 19:36:19 -0800 Subject: [PATCH 02/18] try something --- pandas/_libs/tslibs/offsets.pyx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index dc049e9195d3f..f2ec069c82714 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -441,6 +441,7 @@ cdef class BaseOffset: def __add__(self, other): if not isinstance(self, BaseOffset): # cython semantics; this is __radd__ + # TODO: (Cython 3.0) remove this, this moved to __radd__ return other.__add__(self) elif util.is_array(other) and other.dtype == object: @@ -451,6 +452,9 @@ cdef class BaseOffset: except ApplyTypeError: return NotImplemented + def __radd__(self, other): + return other.__add__(self) + def __sub__(self, other): if PyDateTime_Check(other): raise TypeError('Cannot subtract datetime from offset.') @@ -902,6 +906,7 @@ cdef class Tick(SingleConstructorOffset): def __add__(self, other): if not isinstance(self, Tick): # cython semantics; this is __radd__ + # TODO: (Cython 3.0) remove this, this moved to __radd__ return other.__add__(self) if isinstance(other, Tick): @@ -918,6 +923,8 @@ cdef class Tick(SingleConstructorOffset): raise OverflowError( f"the add operation between {self} and {other} will overflow" ) from err + def __radd__(self, other): + return other.__add__(self) def _apply(self, other): # Timestamp can handle tz and nano sec, thus no need to use apply_wraps From d22e5883f0dd75f81eb3e3019a675bb0d52aa023 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 17 Feb 2022 06:50:02 -0800 Subject: [PATCH 03/18] try something else --- pandas/_libs/tslibs/offsets.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index f2ec069c82714..5608736d0ac7b 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -453,7 +453,7 @@ cdef class BaseOffset: return NotImplemented def __radd__(self, other): - return other.__add__(self) + return self.__add__(other) def __sub__(self, other): if PyDateTime_Check(other): From 646dcedc4fefb99073c699678704d03680d7c958 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 23 Feb 2022 17:55:02 -0800 Subject: [PATCH 04/18] update --- pandas/_libs/tslibs/offsets.pyx | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 5608736d0ac7b..85a8e290b9711 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -453,6 +453,9 @@ cdef class BaseOffset: return NotImplemented def __radd__(self, other): + # We land here because the other object doesn't know how to add + # offset. We can call our own add method since order of operations + # doesn't matter. return self.__add__(other) def __sub__(self, other): @@ -462,12 +465,19 @@ cdef class BaseOffset: return type(self)(self.n - other.n, normalize=self.normalize, **self.kwds) elif not isinstance(self, BaseOffset): + # TODO(Cython 3): remove, this moved to __rsub__ # cython semantics, this is __rsub__ return (-other).__add__(self) else: # e.g. PeriodIndex return NotImplemented + def __rsub__(self, other): + # We land here because the other object doesn't know how to sub + # offset. We can call our own add method since order of operations + # doesn't matter. + return self.__sub__(other) + def __call__(self, other): warnings.warn( "DateOffset.__call__ is deprecated and will be removed in a future " @@ -494,10 +504,17 @@ cdef class BaseOffset: return type(self)(n=other * self.n, normalize=self.normalize, **self.kwds) elif not isinstance(self, BaseOffset): + # TODO: (Cython 3) remove this, this moved to __rmul__ # cython semantics, this is __rmul__ return other.__mul__(self) return NotImplemented + def __rmul__(self, other): + # We land here because the other object doesn't know how to mul + # offset. We can call our own mul method since order of operations + # doesn't matter. + return self.__mul__(other) + def __neg__(self): # Note: we are deferring directly to __mul__ instead of __rmul__, as # that allows us to use methods that can go in a `cdef class` @@ -882,6 +899,7 @@ cdef class Tick(SingleConstructorOffset): def __mul__(self, other): if not isinstance(self, Tick): + # TODO: Cython3, remove this, this moved to __rmul__ # cython semantics, this is __rmul__ return other.__mul__(self) if is_float_object(other): @@ -895,6 +913,12 @@ cdef class Tick(SingleConstructorOffset): return new_self * other return BaseOffset.__mul__(self, other) + def __rmul__(self, other): + # We land here because the other object doesn't know how to mul + # offset. We can call our own mul method since order of operations + # doesn't matter. + return self.__mul__(other) + def __truediv__(self, other): if not isinstance(self, Tick): # cython semantics mean the args are sometimes swapped @@ -924,7 +948,10 @@ cdef class Tick(SingleConstructorOffset): f"the add operation between {self} and {other} will overflow" ) from err def __radd__(self, other): - return other.__add__(self) + # We land here because the other object doesn't know how to add + # offset. We can call our own add method since order of operations + # doesn't matter. + return self.__add__(other) def _apply(self, other): # Timestamp can handle tz and nano sec, thus no need to use apply_wraps From 4a8722779233538b35414072f355237fe88d7bc5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 26 Feb 2022 14:24:14 -0800 Subject: [PATCH 05/18] install cython from master --- ci/deps/actions-310-numpydev.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 2c53da6587425..c686744ecf19e 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -15,7 +15,8 @@ dependencies: - pytz - pip - pip: - - cython + #- cython # TODO: don't install from master after Cython 3.0.0a11 is released + - "git+https://github.com/cython/cython.git@master" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" From d5a9c7e2041a91ab85b31035471c54b2ce21f54d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 27 Feb 2022 09:21:20 -0800 Subject: [PATCH 06/18] try to get further in the test suite --- ci/deps/actions-310-numpydev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index c686744ecf19e..c8967125738cc 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -16,7 +16,7 @@ dependencies: - pip - pip: #- cython # TODO: don't install from master after Cython 3.0.0a11 is released - - "git+https://github.com/cython/cython.git@master" + - "git+https://github.com/da-woods/cython.git@object-none" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" From 5aa9f132f9376e18107755b9fcc160c852e0c322 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 27 Feb 2022 10:43:34 -0800 Subject: [PATCH 07/18] revert back to cython master branch --- ci/deps/actions-310-numpydev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index c8967125738cc..c686744ecf19e 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -16,7 +16,7 @@ dependencies: - pip - pip: #- cython # TODO: don't install from master after Cython 3.0.0a11 is released - - "git+https://github.com/da-woods/cython.git@object-none" + - "git+https://github.com/cython/cython.git@master" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" From de88da99a1837a454fb8bf6f0fbd2e39bad837ad Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 28 Feb 2022 18:22:00 -0800 Subject: [PATCH 08/18] try to get farther in the test suite --- ci/deps/actions-310-numpydev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index c686744ecf19e..1ee233398b8ef 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -16,7 +16,7 @@ dependencies: - pip - pip: #- cython # TODO: don't install from master after Cython 3.0.0a11 is released - - "git+https://github.com/cython/cython.git@master" + - "git+https://github.com/lithomas1/cython.git@debug-pandas" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" From ae3e6fa72cc675e4a8581c9c24eabbda6cd3d420 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 28 Feb 2022 20:58:06 -0800 Subject: [PATCH 09/18] fixes to offsets.pyx --- pandas/_libs/tslibs/offsets.pyx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 85a8e290b9711..b04cf9cb40dd0 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -474,9 +474,8 @@ cdef class BaseOffset: def __rsub__(self, other): # We land here because the other object doesn't know how to sub - # offset. We can call our own add method since order of operations - # doesn't matter. - return self.__sub__(other) + # offset. + return (-self).__add__(other) def __call__(self, other): warnings.warn( @@ -927,6 +926,10 @@ cdef class Tick(SingleConstructorOffset): result = self.delta.__truediv__(other) return _wrap_timedelta_result(result) + def __rtruediv__(self, other): + result = self.delta.__rtruediv__(other) + return _wrap_timedelta_result(result) + def __add__(self, other): if not isinstance(self, Tick): # cython semantics; this is __radd__ From dc7517a96a59b7f597bc3b0c241bcad6afe7d779 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 1 Mar 2022 18:29:55 -0800 Subject: [PATCH 10/18] fix some more binops --- pandas/_libs/tslibs/nattype.pyx | 23 +++++++++++++++++++++++ pandas/_libs/tslibs/period.pyx | 11 +++++++++++ pandas/_libs/tslibs/timestamps.pyx | 9 +++++++++ 3 files changed, 43 insertions(+) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index e6a70177463b8..b9ef6e5e1e450 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -154,6 +154,7 @@ cdef class _NaT(datetime): def __add__(self, other): if self is not c_NaT: + # TODO: Cython 3, remove this it moved to __radd__ # cython __radd__ semantics self, other = other, self @@ -180,6 +181,9 @@ cdef class _NaT(datetime): # Includes Period, DateOffset going through here return NotImplemented + def __radd__(self, other): + return self.__add__(other) + def __sub__(self, other): # Duplicate some logic from _Timestamp.__sub__ to avoid needing # to subclass; allows us to @final(_Timestamp.__sub__) @@ -188,6 +192,7 @@ cdef class _NaT(datetime): if self is not c_NaT: # cython __rsub__ semantics + # TODO: Cython 3, remove __rsub__ logic from here self, other = other, self is_rsub = True @@ -231,6 +236,24 @@ cdef class _NaT(datetime): # Includes Period, DateOffset going through here return NotImplemented + def __rsub__(self, other): + if util.is_array(other): + if other.dtype.kind == "m": + # timedelta64 - NaT we have to treat NaT as timedelta64 + # for this to be meaningful, and the result is timedelta64 + result = np.empty(other.shape, dtype="timedelta64[ns]") + result.fill("NaT") + return result + + elif other.dtype.kind == "M": + # We treat NaT as a datetime, so regardless of whether this is + # NaT - other or other - NaT, the result is timedelta64 + result = np.empty(other.shape, dtype="timedelta64[ns]") + result.fill("NaT") + return result + # other cases are same, swap operands is allowed even though we subtract because this is NaT + return self.__sub__(other) + def __pos__(self): return NaT diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index 8c331b13f9735..24cb1f97c7c7c 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -1739,6 +1739,7 @@ cdef class _Period(PeriodMixin): def __add__(self, other): if not is_period_object(self): # cython semantics; this is analogous to a call to __radd__ + # TODO: Cython3, remove this if self is NaT: return NaT return other.__add__(self) @@ -1763,6 +1764,11 @@ cdef class _Period(PeriodMixin): return NotImplemented + def __radd__(self, other): + if other is NaT: + return NaT + return self.__add__(other) + def __sub__(self, other): if not is_period_object(self): # cython semantics; this is like a call to __rsub__ @@ -1789,6 +1795,11 @@ cdef class _Period(PeriodMixin): return NotImplemented + def __rsub__(self, other): + if other is NaT: + return NaT + return NotImplemented + def asfreq(self, freq, how='E') -> "Period": """ Convert Period to desired frequency, at the start or end of the interval. diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index aad49bd70b120..94a0a57739dc9 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -308,9 +308,13 @@ cdef class _Timestamp(ABCTimestamp): elif not isinstance(self, _Timestamp): # cython semantics, args have been switched and this is __radd__ + # TODO: Cython 3, remove this it moved to __radd__ return other.__add__(self) return NotImplemented + def __radd__(self, other): + return self.__add__(other) + def __sub__(self, other): if is_any_td_scalar(other) or is_integer_object(other): @@ -367,10 +371,15 @@ cdef class _Timestamp(ABCTimestamp): elif is_datetime64_object(self): # GH#28286 cython semantics for __rsub__, `other` is actually # the Timestamp + # TODO: Cython 3 remove this, this moved to __rsub__ return type(other)(self) - other return NotImplemented + def __rsub__(self, other): + if is_datetime64_object(other): + return type(self)(other) - self + return NotImplemented # ----------------------------------------------------------------- cdef int64_t _maybe_convert_value_to_local(self): From 416bca57e18823e93502bede6c72ea6b9f8cb39d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 5 Mar 2022 14:41:17 -0800 Subject: [PATCH 11/18] fix some more tests --- pandas/_libs/interval.pyx | 14 ++++++++++++++ pandas/_libs/tslibs/timestamps.pyx | 7 +++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/interval.pyx b/pandas/_libs/interval.pyx index aba635e19995a..4b515f183e347 100644 --- a/pandas/_libs/interval.pyx +++ b/pandas/_libs/interval.pyx @@ -414,6 +414,15 @@ cdef class Interval(IntervalMixin): return Interval(y.left + self, y.right + self, closed=y.closed) return NotImplemented + def __radd__(self, y): + if ( + isinstance(y, numbers.Number) + or PyDelta_Check(y) + or is_timedelta64_object(y) + ): + return self.__add__(y) + return NotImplemented + def __sub__(self, y): if ( isinstance(y, numbers.Number) @@ -430,6 +439,11 @@ cdef class Interval(IntervalMixin): return Interval(y.left * self, y.right * self, closed=y.closed) return NotImplemented + def __rmul__(self, y): + if isinstance(y, numbers.Number): + return self.__mul__(y) + return NotImplemented + def __truediv__(self, y): if isinstance(y, numbers.Number): return Interval(self.left / y, self.right / y, closed=self.closed) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 94a0a57739dc9..c358d51f97cc4 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -313,7 +313,10 @@ cdef class _Timestamp(ABCTimestamp): return NotImplemented def __radd__(self, other): - return self.__add__(other) + # Have to duplicate checks to avoid infinite recursion due to NotImplemented + if is_any_td_scalar(other) or is_integer_object(other) or is_array(other): + return self.__add__(other) + return NotImplemented def __sub__(self, other): @@ -377,7 +380,7 @@ cdef class _Timestamp(ABCTimestamp): return NotImplemented def __rsub__(self, other): - if is_datetime64_object(other): + if PyDateTime_Check(other) or is_datetime64_object(other): return type(self)(other) - self return NotImplemented # ----------------------------------------------------------------- From 127e97b9fb76ca2563a490209a4ad7a23a0d3b76 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 7 Mar 2022 20:06:48 -0800 Subject: [PATCH 12/18] workaround cython bug? --- pandas/_libs/tslibs/timestamps.pyx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index c358d51f97cc4..757045619a55d 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -282,8 +282,10 @@ cdef class _Timestamp(ABCTimestamp): return other.tzinfo is None def __add__(self, other): - cdef: - int64_t nanos = 0 + # TODO: There is a Cython 3 bug where self.values + nanos + # would silently overflow if this is defined + #cdef: + # int64_t nanos = 0 if is_any_td_scalar(other): nanos = delta_to_nanoseconds(other) @@ -344,6 +346,7 @@ cdef class _Timestamp(ABCTimestamp): and (PyDateTime_Check(other) or is_datetime64_object(other))): # both_timestamps is to determine whether Timedelta(self - other) # should raise the OOB error, or fall back returning a timedelta. + # TODO: Cython 3: clean out the bits that moved to __rsub__ both_timestamps = (isinstance(other, _Timestamp) and isinstance(self, _Timestamp)) if isinstance(self, _Timestamp): @@ -380,7 +383,14 @@ cdef class _Timestamp(ABCTimestamp): return NotImplemented def __rsub__(self, other): - if PyDateTime_Check(other) or is_datetime64_object(other): + if PyDateTime_Check(other): + try: + return type(self)(other) - self + except (OverflowError, OutOfBoundsDatetime) as err: + # We get here in stata tests, fall back to stdlib datetime + # method and return stdlib timedelta object + pass + elif is_datetime64_object(other): return type(self)(other) - self return NotImplemented # ----------------------------------------------------------------- From 7259c4cca5a86187e15c11ac8548b0f89069fd5d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 14 Mar 2022 13:01:31 -0700 Subject: [PATCH 13/18] standardize todos --- pandas/_libs/tslibs/nattype.pyx | 4 ++-- pandas/_libs/tslibs/offsets.pyx | 12 +++--------- pandas/_libs/tslibs/period.pyx | 2 +- pandas/_libs/tslibs/timestamps.pyx | 6 +++--- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index b9ef6e5e1e450..8e1b86182f2bc 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -154,7 +154,7 @@ cdef class _NaT(datetime): def __add__(self, other): if self is not c_NaT: - # TODO: Cython 3, remove this it moved to __radd__ + # TODO(cython3): remove this it moved to __radd__ # cython __radd__ semantics self, other = other, self @@ -192,7 +192,7 @@ cdef class _NaT(datetime): if self is not c_NaT: # cython __rsub__ semantics - # TODO: Cython 3, remove __rsub__ logic from here + # TODO(cython3): remove __rsub__ logic from here self, other = other, self is_rsub = True diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index b04cf9cb40dd0..e16bb4c0a6cac 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -441,7 +441,7 @@ cdef class BaseOffset: def __add__(self, other): if not isinstance(self, BaseOffset): # cython semantics; this is __radd__ - # TODO: (Cython 3.0) remove this, this moved to __radd__ + # TODO(cython3): remove this, this moved to __radd__ return other.__add__(self) elif util.is_array(other) and other.dtype == object: @@ -503,7 +503,7 @@ cdef class BaseOffset: return type(self)(n=other * self.n, normalize=self.normalize, **self.kwds) elif not isinstance(self, BaseOffset): - # TODO: (Cython 3) remove this, this moved to __rmul__ + # TODO(cython3): remove this, this moved to __rmul__ # cython semantics, this is __rmul__ return other.__mul__(self) return NotImplemented @@ -913,9 +913,6 @@ cdef class Tick(SingleConstructorOffset): return BaseOffset.__mul__(self, other) def __rmul__(self, other): - # We land here because the other object doesn't know how to mul - # offset. We can call our own mul method since order of operations - # doesn't matter. return self.__mul__(other) def __truediv__(self, other): @@ -933,7 +930,7 @@ cdef class Tick(SingleConstructorOffset): def __add__(self, other): if not isinstance(self, Tick): # cython semantics; this is __radd__ - # TODO: (Cython 3.0) remove this, this moved to __radd__ + # TODO(cython3): remove this, this moved to __radd__ return other.__add__(self) if isinstance(other, Tick): @@ -951,9 +948,6 @@ cdef class Tick(SingleConstructorOffset): f"the add operation between {self} and {other} will overflow" ) from err def __radd__(self, other): - # We land here because the other object doesn't know how to add - # offset. We can call our own add method since order of operations - # doesn't matter. return self.__add__(other) def _apply(self, other): diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index ecce5e4f2adeb..64596ebe710ab 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -1720,7 +1720,7 @@ cdef class _Period(PeriodMixin): def __add__(self, other): if not is_period_object(self): # cython semantics; this is analogous to a call to __radd__ - # TODO: Cython3, remove this + # TODO(cython3): remove this if self is NaT: return NaT return other.__add__(self) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 757045619a55d..e115f4b7c76b6 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -310,7 +310,7 @@ cdef class _Timestamp(ABCTimestamp): elif not isinstance(self, _Timestamp): # cython semantics, args have been switched and this is __radd__ - # TODO: Cython 3, remove this it moved to __radd__ + # TODO(cython3): remove this it moved to __radd__ return other.__add__(self) return NotImplemented @@ -346,7 +346,7 @@ cdef class _Timestamp(ABCTimestamp): and (PyDateTime_Check(other) or is_datetime64_object(other))): # both_timestamps is to determine whether Timedelta(self - other) # should raise the OOB error, or fall back returning a timedelta. - # TODO: Cython 3: clean out the bits that moved to __rsub__ + # TODO(cython3): clean out the bits that moved to __rsub__ both_timestamps = (isinstance(other, _Timestamp) and isinstance(self, _Timestamp)) if isinstance(self, _Timestamp): @@ -377,7 +377,7 @@ cdef class _Timestamp(ABCTimestamp): elif is_datetime64_object(self): # GH#28286 cython semantics for __rsub__, `other` is actually # the Timestamp - # TODO: Cython 3 remove this, this moved to __rsub__ + # TODO(cython3): remove this, this moved to __rsub__ return type(other)(self) - other return NotImplemented From f123a47ff420c821291652f2cd2849b94670b480 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 15 Mar 2022 18:10:21 -0700 Subject: [PATCH 14/18] switch to master branch --- ci/deps/actions-310-numpydev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 1ee233398b8ef..c686744ecf19e 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -16,7 +16,7 @@ dependencies: - pip - pip: #- cython # TODO: don't install from master after Cython 3.0.0a11 is released - - "git+https://github.com/lithomas1/cython.git@debug-pandas" + - "git+https://github.com/cython/cython.git@master" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" From 00c492de53a1b31007acd5a1cea15d37a4c1fe60 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:56:38 -0700 Subject: [PATCH 15/18] address comments --- pandas/_libs/interval.pyx | 20 ++++++++++++-------- pandas/_libs/tslibs/nattype.pyx | 2 ++ pandas/_libs/tslibs/offsets.pyx | 11 ++--------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pandas/_libs/interval.pyx b/pandas/_libs/interval.pyx index 4b515f183e347..f31eb6a117a03 100644 --- a/pandas/_libs/interval.pyx +++ b/pandas/_libs/interval.pyx @@ -404,6 +404,8 @@ cdef class Interval(IntervalMixin): ): return Interval(self.left + y, self.right + y, closed=self.closed) elif ( + # __radd__ pattern + # TODO(cython3): remove this isinstance(y, Interval) and ( isinstance(self, numbers.Number) @@ -414,13 +416,13 @@ cdef class Interval(IntervalMixin): return Interval(y.left + self, y.right + self, closed=y.closed) return NotImplemented - def __radd__(self, y): + def __radd__(self, other): if ( - isinstance(y, numbers.Number) - or PyDelta_Check(y) - or is_timedelta64_object(y) + isinstance(other, numbers.Number) + or PyDelta_Check(other) + or is_timedelta64_object(other) ): - return self.__add__(y) + return Interval(self.left + other, self.right + other, closed=self.closed) return NotImplemented def __sub__(self, y): @@ -436,12 +438,14 @@ cdef class Interval(IntervalMixin): if isinstance(y, numbers.Number): return Interval(self.left * y, self.right * y, closed=self.closed) elif isinstance(y, Interval) and isinstance(self, numbers.Number): + # __radd__ semantics + # TODO(cython3): remove this return Interval(y.left * self, y.right * self, closed=y.closed) return NotImplemented - def __rmul__(self, y): - if isinstance(y, numbers.Number): - return self.__mul__(y) + def __rmul__(self, other): + if isinstance(other, numbers.Number): + return Interval(self.left * other, self.right * other, closed=self.closed) return NotImplemented def __truediv__(self, y): diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 8e1b86182f2bc..d2ea2a4925a2c 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -216,6 +216,8 @@ cdef class _NaT(datetime): result.fill("NaT") return result + # __rsub__ logic here + # TODO(cython3): remove this, move above code out of ``if not is_rsub`` block # timedelta64 - NaT we have to treat NaT as timedelta64 # for this to be meaningful, and the result is timedelta64 result = np.empty(other.shape, dtype="timedelta64[ns]") diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index e16bb4c0a6cac..86ddd5c502023 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -453,9 +453,6 @@ cdef class BaseOffset: return NotImplemented def __radd__(self, other): - # We land here because the other object doesn't know how to add - # offset. We can call our own add method since order of operations - # doesn't matter. return self.__add__(other) def __sub__(self, other): @@ -473,8 +470,6 @@ cdef class BaseOffset: return NotImplemented def __rsub__(self, other): - # We land here because the other object doesn't know how to sub - # offset. return (-self).__add__(other) def __call__(self, other): @@ -509,9 +504,6 @@ cdef class BaseOffset: return NotImplemented def __rmul__(self, other): - # We land here because the other object doesn't know how to mul - # offset. We can call our own mul method since order of operations - # doesn't matter. return self.__mul__(other) def __neg__(self): @@ -898,7 +890,7 @@ cdef class Tick(SingleConstructorOffset): def __mul__(self, other): if not isinstance(self, Tick): - # TODO: Cython3, remove this, this moved to __rmul__ + # TODO(cython3), remove this, this moved to __rmul__ # cython semantics, this is __rmul__ return other.__mul__(self) if is_float_object(other): @@ -947,6 +939,7 @@ cdef class Tick(SingleConstructorOffset): raise OverflowError( f"the add operation between {self} and {other} will overflow" ) from err + def __radd__(self, other): return self.__add__(other) From 92325d7c55ea238c64c26ed33aecab70d607c9ab Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 22 Mar 2022 09:52:13 -0700 Subject: [PATCH 16/18] handle overflow correctly --- pandas/_libs/tslibs/timestamps.pyx | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 71170e4b55a70..69a98f57b1da5 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -281,15 +281,18 @@ cdef class _Timestamp(ABCTimestamp): return other.tzinfo is not None return other.tzinfo is None + @cython.overflowcheck(True) def __add__(self, other): - # TODO: There is a Cython 3 bug where self.values + nanos - # would silently overflow if this is defined - #cdef: - # int64_t nanos = 0 + cdef: + int64_t nanos = 0 if is_any_td_scalar(other): nanos = delta_to_nanoseconds(other) - result = type(self)(self.value + nanos, tz=self.tzinfo) + try: + result = type(self)(self.value + nanos, tz=self.tzinfo) + except OverflowError: + # Use Python ints + result = type(self)(int(self.value) + int(nanos), tz=self.tzinfo) if result is not NaT: result._set_freq(self._freq) # avoid warning in constructor return result From eaea742db66aea5fadfbae5331fe50dd6df5e751 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 24 Mar 2022 20:03:35 -0700 Subject: [PATCH 17/18] address comments --- pandas/_libs/tslibs/offsets.pyx | 2 +- pandas/_libs/tslibs/timestamps.pyx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index fb46b277b6d52..5a7f931292e7b 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -471,7 +471,7 @@ cdef class BaseOffset: return type(self)(self.n - other.n, normalize=self.normalize, **self.kwds) elif not isinstance(self, BaseOffset): - # TODO(Cython 3): remove, this moved to __rsub__ + # TODO(cython3): remove, this moved to __rsub__ # cython semantics, this is __rsub__ return (-other).__add__(self) else: diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 69a98f57b1da5..7cb52d6cd3527 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -284,7 +284,7 @@ cdef class _Timestamp(ABCTimestamp): @cython.overflowcheck(True) def __add__(self, other): cdef: - int64_t nanos = 0 + int64_t nanos = 0 if is_any_td_scalar(other): nanos = delta_to_nanoseconds(other) @@ -292,6 +292,7 @@ cdef class _Timestamp(ABCTimestamp): result = type(self)(self.value + nanos, tz=self.tzinfo) except OverflowError: # Use Python ints + # Hit in test_tdi_add_overflow result = type(self)(int(self.value) + int(nanos), tz=self.tzinfo) if result is not NaT: result._set_freq(self._freq) # avoid warning in constructor From f4674520c54831b867b820d708b6d7c86bf58308 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 24 Mar 2022 20:41:53 -0700 Subject: [PATCH 18/18] test something --- pandas/_libs/tslibs/period.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index 9117f2cf13b50..c18d71e324a28 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -1736,13 +1736,12 @@ cdef class _Period(PeriodMixin): return NotImplemented def __radd__(self, other): - if other is NaT: - return NaT return self.__add__(other) def __sub__(self, other): if not is_period_object(self): # cython semantics; this is like a call to __rsub__ + # TODO(cython3): remove this if self is NaT: return NaT return NotImplemented