From a3bedb08bc97375a5af3c90686c5d7998c57d203 Mon Sep 17 00:00:00 2001 From: droussea2001 <19688507+droussea2001@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:05:03 +0200 Subject: [PATCH 1/5] *TST 54315 Add tests for pd.Timedelta and pd.NA div / truediv --- .../tests/scalar/timedelta/test_timedelta.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pandas/tests/scalar/timedelta/test_timedelta.py b/pandas/tests/scalar/timedelta/test_timedelta.py index 73b2da0f7dd50..23aad79473733 100644 --- a/pandas/tests/scalar/timedelta/test_timedelta.py +++ b/pandas/tests/scalar/timedelta/test_timedelta.py @@ -15,6 +15,8 @@ NaT, iNaT, ) +from pandas._libs.missing import NA + from pandas._libs.tslibs.dtypes import NpyDatetimeUnit from pandas.errors import OutOfBoundsTimedelta @@ -138,6 +140,15 @@ def test_truediv_numeric(self, td): assert res._value == td._value / 2 assert res._creso == td._creso + def test_truediv_na_type_not_supported(self, td): + msg_td_floordiv_na = r"unsupported operand type\(s\) for /: 'Timedelta' and 'NAType'" + with pytest.raises(TypeError, match=msg_td_floordiv_na): + td / NA + + msg_na_floordiv_td = r"unsupported operand type\(s\) for /: 'NAType' and 'Timedelta'" + with pytest.raises(TypeError, match=msg_na_floordiv_td): + NA / td + def test_floordiv_timedeltalike(self, td): assert td // td == 1 assert (2.5 * td) // td == 2 @@ -161,6 +172,7 @@ def test_floordiv_timedeltalike(self, td): result = right // left assert result == 0 + def test_floordiv_numeric(self, td): assert td // np.nan is NaT @@ -182,6 +194,15 @@ def test_floordiv_numeric(self, td): assert res._value == td._value // 2 assert res._creso == td._creso + def test_floordiv_na_type_not_supported(self, td): + msg_td_floordiv_na = r"unsupported operand type\(s\) for //: 'Timedelta' and 'NAType'" + with pytest.raises(TypeError, match=msg_td_floordiv_na): + td // NA + + msg_na_floordiv_td = r"unsupported operand type\(s\) for //: 'NAType' and 'Timedelta'" + with pytest.raises(TypeError, match=msg_na_floordiv_td): + NA // td + def test_addsub_mismatched_reso(self, td): # need to cast to since td is out of bounds for ns, so # so we would raise OverflowError without casting From 6ec91c03b6aa367d46d9346972ccf677796c9fc6 Mon Sep 17 00:00:00 2001 From: droussea2001 <19688507+droussea2001@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:34:56 +0200 Subject: [PATCH 2/5] Add release doc for TST 54315 --- doc/source/whatsnew/v3.0.0.rst | 94 +++++++--------------------------- 1 file changed, 18 insertions(+), 76 deletions(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index c0a2dc7e39f29..ac7a49a3f2cd4 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -34,7 +34,6 @@ Other enhancements - Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`) - Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`) - Users can globally disable any ``PerformanceWarning`` by setting the option ``mode.performance_warnings`` to ``False`` (:issue:`56920`) -- :meth:`Styler.format_index_names` can now be used to format the index and column names (:issue:`48936` and :issue:`47489`) - .. --------------------------------------------------------------------------- @@ -92,16 +91,16 @@ Now using multiple groupings will also pass the unobserved groups to the provide Similarly: -- In previous versions of pandas the method :meth:`.DataFrameGroupBy.sum` would result in ``0`` for unobserved groups, but :meth:`.DataFrameGroupBy.prod`, :meth:`.DataFrameGroupBy.all`, and :meth:`.DataFrameGroupBy.any` would all result in NA values. Now these methods result in ``1``, ``True``, and ``False`` respectively. -- :meth:`.DataFrameGroupBy.groups` did not include unobserved groups and now does. + - In previous versions of pandas the method :meth:`.DataFrameGroupBy.sum` would result in ``0`` for unobserved groups, but :meth:`.DataFrameGroupBy.prod`, :meth:`.DataFrameGroupBy.all`, and :meth:`.DataFrameGroupBy.any` would all result in NA values. Now these methods result in ``1``, ``True``, and ``False`` respectively. + - :meth:`.DataFrameGroupBy.groups` did not include unobserved groups and now does. These improvements also fixed certain bugs in groupby: -- :meth:`.DataFrameGroupBy.agg` would fail when there are multiple groupings, unobserved groups, and ``as_index=False`` (:issue:`36698`) -- :meth:`.DataFrameGroupBy.groups` with ``sort=False`` would sort groups; they now occur in the order they are observed (:issue:`56966`) -- :meth:`.DataFrameGroupBy.nunique` would fail when there are multiple groupings, unobserved groups, and ``as_index=False`` (:issue:`52848`) -- :meth:`.DataFrameGroupBy.sum` would have incorrect values when there are multiple groupings, unobserved groups, and non-numeric data (:issue:`43891`) -- :meth:`.DataFrameGroupBy.value_counts` would produce incorrect results when used with some categorical and some non-categorical groupings and ``observed=False`` (:issue:`56016`) + - :meth:`.DataFrameGroupBy.nunique` would fail when there are multiple groupings, unobserved groups, and ``as_index=False`` (:issue:`52848`) + - :meth:`.DataFrameGroupBy.agg` would fail when there are multiple groupings, unobserved groups, and ``as_index=False`` (:issue:`36698`) + - :meth:`.DataFrameGroupBy.sum` would have incorrect values when there are multiple groupings, unobserved groups, and non-numeric data (:issue:`43891`) + - :meth:`.DataFrameGroupBy.groups` with ``sort=False`` would sort groups; they now occur in the order they are observed (:issue:`56966`) + - :meth:`.DataFrameGroupBy.value_counts` would produce incorrect results when used with some categorical and some non-categorical groupings and ``observed=False`` (:issue:`56016`) .. _whatsnew_300.notable_bug_fixes.notable_bug_fix2: @@ -131,13 +130,11 @@ For `optional libraries ` (:issue:`54710`) - In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`) - Iterating over a :class:`.DataFrameGroupBy` or :class:`.SeriesGroupBy` will return tuples of length 1 for the groups when grouping by ``level`` a list of length 1 (:issue:`50064`) @@ -278,7 +241,6 @@ Removal of prior version deprecations/changes - Removed argument ``limit`` from :meth:`DataFrame.pct_change`, :meth:`Series.pct_change`, :meth:`.DataFrameGroupBy.pct_change`, and :meth:`.SeriesGroupBy.pct_change`; the argument ``method`` must be set to ``None`` and will be removed in a future version of pandas (:issue:`53520`) - Removed deprecated argument ``obj`` in :meth:`.DataFrameGroupBy.get_group` and :meth:`.SeriesGroupBy.get_group` (:issue:`53545`) - Removed deprecated behavior of :meth:`Series.agg` using :meth:`Series.apply` (:issue:`53325`) -- Removed deprecated keyword ``method`` on :meth:`Series.fillna`, :meth:`DataFrame.fillna` (:issue:`57760`) - Removed option ``mode.use_inf_as_na``, convert inf entries to ``NaN`` before instead (:issue:`51684`) - Removed support for :class:`DataFrame` in :meth:`DataFrame.from_records`(:issue:`51697`) - Removed support for ``errors="ignore"`` in :func:`to_datetime`, :func:`to_timedelta` and :func:`to_numeric` (:issue:`55734`) @@ -290,10 +252,7 @@ Removal of prior version deprecations/changes - Removed unused arguments ``*args`` and ``**kwargs`` in :class:`Resampler` methods (:issue:`50977`) - Unrecognized timezones when parsing strings to datetimes now raises a ``ValueError`` (:issue:`51477`) - Removed the :class:`Grouper` attributes ``ax``, ``groups``, ``indexer``, and ``obj`` (:issue:`51206`, :issue:`51182`) -- Removed deprecated keyword ``verbose`` on :func:`read_csv` and :func:`read_table` (:issue:`56556`) -- Removed the ``method`` keyword in ``ExtensionArray.fillna``, implement ``ExtensionArray._pad_or_backfill`` instead (:issue:`53621`) - Removed the attribute ``dtypes`` from :class:`.DataFrameGroupBy` (:issue:`51997`) -- Enforced deprecation of ``argmin``, ``argmax``, ``idxmin``, and ``idxmax`` returning a result when ``skipna=False`` and an NA value is encountered or all values are NA values; these operations will now raise in such cases (:issue:`33941`, :issue:`51276`) .. --------------------------------------------------------------------------- .. _whatsnew_300.performance: @@ -301,9 +260,6 @@ Removal of prior version deprecations/changes Performance improvements ~~~~~~~~~~~~~~~~~~~~~~~~ - :attr:`Categorical.categories` returns a :class:`RangeIndex` columns instead of an :class:`Index` if the constructed ``values`` was a ``range``. (:issue:`57787`) -- :class:`DataFrame` returns a :class:`RangeIndex` columns when possible when ``data`` is a ``dict`` (:issue:`57943`) -- :class:`Series` returns a :class:`RangeIndex` index when possible when ``data`` is a ``dict`` (:issue:`58118`) -- :func:`concat` returns a :class:`RangeIndex` column when possible when ``objs`` contains :class:`Series` and :class:`DataFrame` and ``axis=0`` (:issue:`58119`) - :func:`concat` returns a :class:`RangeIndex` level in the :class:`MultiIndex` result when ``keys`` is a ``range`` or :class:`RangeIndex` (:issue:`57542`) - :meth:`RangeIndex.append` returns a :class:`RangeIndex` instead of a :class:`Index` when appending values that could continue the :class:`RangeIndex` (:issue:`57467`) - :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57542`) @@ -315,20 +271,14 @@ Performance improvements - Performance improvement in :meth:`DataFrameGroupBy.ffill`, :meth:`DataFrameGroupBy.bfill`, :meth:`SeriesGroupBy.ffill`, and :meth:`SeriesGroupBy.bfill` (:issue:`56902`) - Performance improvement in :meth:`Index.join` by propagating cached attributes in cases where the result matches one of the inputs (:issue:`57023`) - Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`) -- Performance improvement in :meth:`Index.to_frame` returning a :class:`RangeIndex` columns of a :class:`Index` when possible. (:issue:`58018`) - Performance improvement in :meth:`MultiIndex.equals` for equal length indexes (:issue:`56990`) -- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask or integers returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`) +- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`) - Performance improvement in :meth:`RangeIndex.append` when appending the same index (:issue:`57252`) -- Performance improvement in :meth:`RangeIndex.argmin` and :meth:`RangeIndex.argmax` (:issue:`57823`) -- Performance improvement in :meth:`RangeIndex.insert` returning a :class:`RangeIndex` instead of a :class:`Index` when the :class:`RangeIndex` is empty. (:issue:`57833`) -- Performance improvement in :meth:`RangeIndex.round` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57824`) -- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`, :issue:`57752`) -- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`, :issue:`57752`) -- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`, :issue:`57752`) -- Performance improvement in :func:`merge` if hash-join can be used (:issue:`57970`) +- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`) +- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`) +- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`) - Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`) - Performance improvement in indexing operations for string dtypes (:issue:`56997`) -- Performance improvement in unary methods on a :class:`RangeIndex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57825`) .. --------------------------------------------------------------------------- .. _whatsnew_300.bug_fixes: @@ -336,16 +286,11 @@ Performance improvements Bug fixes ~~~~~~~~~ - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`) -- Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`) -- Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`) - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`) - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`) -- Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`) - Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`) -- Fixed bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`) - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`) - Fixed bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`) -- Fixed bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`) Categorical ^^^^^^^^^^^ @@ -355,12 +300,12 @@ Categorical Datetimelike ^^^^^^^^^^^^ - Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`) -- Bug in :func:`date_range` where using a negative frequency value would not include all points between the start and end values (:issue:`56382`) - Timedelta ^^^^^^^^^ - Accuracy improvement in :meth:`Timedelta.to_pytimedelta` to round microseconds consistently for large nanosecond based Timedelta (:issue:`57841`) +- Add a test for div and true div between ``Timedelta`` and ``pd.NA`` raising a TypeError (:issue:`54315`) - Timezones @@ -406,7 +351,6 @@ MultiIndex I/O ^^^ - Bug in :meth:`DataFrame.to_excel` when writing empty :class:`DataFrame` with :class:`MultiIndex` on both axes (:issue:`57696`) -- Now all ``Mapping`` s are pretty printed correctly. Before only literal ``dict`` s were. (:issue:`57915`) - - @@ -417,7 +361,7 @@ Period Plotting ^^^^^^^^ -- Bug in :meth:`.DataFrameGroupBy.boxplot` failed when there were multiple groupings (:issue:`14701`) +- - Groupby/resample/rolling @@ -450,10 +394,8 @@ Other ^^^^^ - Bug in :class:`DataFrame` when passing a ``dict`` with a NA scalar and ``columns`` that would always return ``np.nan`` (:issue:`57205`) - Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`) -- Bug in :func:`unique` on :class:`Index` not always returning :class:`Index` (:issue:`57043`) - Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` and ``ascending=False`` not returning a :class:`RangeIndex` columns (:issue:`57293`) - Bug in :meth:`DataFrame.where` where using a non-bool type array in the function would return a ``ValueError`` instead of a ``TypeError`` (:issue:`56330`) -- Bug in :meth:`Index.sort_values` when passing a key function that turns values into tuples, e.g. ``key=natsort.natsort_key``, would raise ``TypeError`` (:issue:`56081`) - Bug in Dataframe Interchange Protocol implementation was returning incorrect results for data buffers' associated dtype, for string and datetime columns (:issue:`54781`) .. ***DO NOT USE THIS SECTION*** From 129cf82b505c4d5e7a36d1a6e1c87e0925b54c32 Mon Sep 17 00:00:00 2001 From: droussea2001 <19688507+droussea2001@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:54:06 +0200 Subject: [PATCH 3/5] Apply pre commit correction for TST 54315 --- .../tests/scalar/timedelta/test_timedelta.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pandas/tests/scalar/timedelta/test_timedelta.py b/pandas/tests/scalar/timedelta/test_timedelta.py index 23aad79473733..01e7ba52e58aa 100644 --- a/pandas/tests/scalar/timedelta/test_timedelta.py +++ b/pandas/tests/scalar/timedelta/test_timedelta.py @@ -11,12 +11,11 @@ import pytest from pandas._libs import lib +from pandas._libs.missing import NA from pandas._libs.tslibs import ( NaT, iNaT, ) -from pandas._libs.missing import NA - from pandas._libs.tslibs.dtypes import NpyDatetimeUnit from pandas.errors import OutOfBoundsTimedelta @@ -141,11 +140,15 @@ def test_truediv_numeric(self, td): assert res._creso == td._creso def test_truediv_na_type_not_supported(self, td): - msg_td_floordiv_na = r"unsupported operand type\(s\) for /: 'Timedelta' and 'NAType'" + msg_td_floordiv_na = ( + r"unsupported operand type\(s\) for /: 'Timedelta' and 'NAType'" + ) with pytest.raises(TypeError, match=msg_td_floordiv_na): td / NA - msg_na_floordiv_td = r"unsupported operand type\(s\) for /: 'NAType' and 'Timedelta'" + msg_na_floordiv_td = ( + r"unsupported operand type\(s\) for /: 'NAType' and 'Timedelta'" + ) with pytest.raises(TypeError, match=msg_na_floordiv_td): NA / td @@ -172,7 +175,6 @@ def test_floordiv_timedeltalike(self, td): result = right // left assert result == 0 - def test_floordiv_numeric(self, td): assert td // np.nan is NaT @@ -195,11 +197,15 @@ def test_floordiv_numeric(self, td): assert res._creso == td._creso def test_floordiv_na_type_not_supported(self, td): - msg_td_floordiv_na = r"unsupported operand type\(s\) for //: 'Timedelta' and 'NAType'" + msg_td_floordiv_na = ( + r"unsupported operand type\(s\) for //: 'Timedelta' and 'NAType'" + ) with pytest.raises(TypeError, match=msg_td_floordiv_na): td // NA - msg_na_floordiv_td = r"unsupported operand type\(s\) for //: 'NAType' and 'Timedelta'" + msg_na_floordiv_td = ( + r"unsupported operand type\(s\) for //: 'NAType' and 'Timedelta'" + ) with pytest.raises(TypeError, match=msg_na_floordiv_td): NA // td From d70a91508ad5d31ee51187179613b93f281efbe1 Mon Sep 17 00:00:00 2001 From: droussea2001 <19688507+droussea2001@users.noreply.github.com> Date: Mon, 8 Apr 2024 22:26:14 +0200 Subject: [PATCH 4/5] "Correct bug fix description removed" --- doc/source/whatsnew/v3.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index f90db21f360cf..6ac3e293a2afd 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -338,6 +338,7 @@ Bug fixes - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`) - Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`) - Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`) +- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using NumPy attributes via ``@`` notation, e.g., ``df.eval("@np.floor(a)")``. (:issue:`58041`) - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`) - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`) - Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`) From cb4dce485e423f5d62700129087a4ab21ce6b7f9 Mon Sep 17 00:00:00 2001 From: droussea2001 <19688507+droussea2001@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:28:26 +0200 Subject: [PATCH 5/5] Remove unnecessary unit test description --- doc/source/whatsnew/v3.0.0.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 6ac3e293a2afd..19b448a1871c2 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -362,7 +362,6 @@ Datetimelike Timedelta ^^^^^^^^^ - Accuracy improvement in :meth:`Timedelta.to_pytimedelta` to round microseconds consistently for large nanosecond based Timedelta (:issue:`57841`) -- Add a test for div and true div between ``Timedelta`` and ``pd.NA`` raising a TypeError (:issue:`54315`) - Timezones