From ec695c781f82d8172630a37b531ed634fbfb91a1 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:42:59 -0700 Subject: [PATCH 1/5] DEPS: Address numpy deprecation of len 1 arrays assignment --- pandas/core/internals/base.py | 4 ++++ pandas/tests/groupby/test_groupby_dropna.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/base.py b/pandas/core/internals/base.py index 523dee97a3c5c..922b3caf87cb9 100644 --- a/pandas/core/internals/base.py +++ b/pandas/core/internals/base.py @@ -187,6 +187,10 @@ def setitem_inplace(self, indexer, value) -> None: # dt64/td64, which do their own validation. value = np_can_hold_element(arr.dtype, value) + if isinstance(value, np.ndarray) and value.ndim == 1 and len(value) == 1: + # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 + value = value[0] + arr[indexer] = value def grouped_reduce(self, func): diff --git a/pandas/tests/groupby/test_groupby_dropna.py b/pandas/tests/groupby/test_groupby_dropna.py index 48cc1518937b3..551642ff64fd5 100644 --- a/pandas/tests/groupby/test_groupby_dropna.py +++ b/pandas/tests/groupby/test_groupby_dropna.py @@ -627,7 +627,7 @@ def test_categorical_transformers( result = getattr(gb_keepna, transformation_func)(*args) expected = getattr(gb_dropna, transformation_func)(*args) for iloc, value in zip( - df[df["x"].isnull()].index.tolist(), null_group_result.values + df[df["x"].isnull()].index.tolist(), null_group_result.values.ravel() ): if expected.ndim == 1: expected.iloc[iloc] = value From 675ade379bb3c2336da0baf3b9dae03139321f89 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 25 Apr 2023 11:00:14 -0700 Subject: [PATCH 2/5] Address other failures, and investigate csv failure --- pandas/_testing/_warnings.py | 2 +- pandas/core/indexing.py | 7 +++++++ pandas/core/internals/blocks.py | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pandas/_testing/_warnings.py b/pandas/_testing/_warnings.py index 201aa81183301..d5e1fbf029069 100644 --- a/pandas/_testing/_warnings.py +++ b/pandas/_testing/_warnings.py @@ -83,7 +83,7 @@ class for all warnings. To raise multiple types of exceptions, ..warn:: This is *not* thread-safe. """ - __tracebackhide__ = True + # __tracebackhide__ = True with warnings.catch_warnings(record=True) as w: warnings.simplefilter(filter_level) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 6aecfe5267e0c..93467c52d742e 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1753,6 +1753,13 @@ def _setitem_with_indexer(self, indexer, value, name: str = "iloc"): if not isinstance(value, ABCSeries): # if not Series (in which case we need to align), # we can short-circuit + if ( + isinstance(value, np.ndarray) + and value.ndim == 1 + and len(value) == 1 + ): + # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 + arr = arr[0] empty_value[indexer[0]] = arr self.obj[key] = empty_value return diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index ff022cb047f3d..f03f88306f8b2 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1063,7 +1063,9 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block: self = self.make_block_same_class( values.T if values.ndim == 2 else values ) - + if isinstance(value, np.ndarray) and value.ndim == 1 and len(value) == 1: + # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 + casted = casted[0] values[indexer] = casted return self From 9e5551e6fbf574d1eef8643deb14755d1281f2a0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 25 Apr 2023 15:18:22 -0700 Subject: [PATCH 3/5] Address csv error, undo one fix --- pandas/_testing/_warnings.py | 2 +- pandas/core/indexing.py | 6 +++--- pandas/core/internals/blocks.py | 3 --- pandas/io/parsers/base_parser.py | 10 +++++++++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pandas/_testing/_warnings.py b/pandas/_testing/_warnings.py index d5e1fbf029069..201aa81183301 100644 --- a/pandas/_testing/_warnings.py +++ b/pandas/_testing/_warnings.py @@ -83,7 +83,7 @@ class for all warnings. To raise multiple types of exceptions, ..warn:: This is *not* thread-safe. """ - # __tracebackhide__ = True + __tracebackhide__ = True with warnings.catch_warnings(record=True) as w: warnings.simplefilter(filter_level) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 93467c52d742e..f97f92f4cb006 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1754,9 +1754,9 @@ def _setitem_with_indexer(self, indexer, value, name: str = "iloc"): # if not Series (in which case we need to align), # we can short-circuit if ( - isinstance(value, np.ndarray) - and value.ndim == 1 - and len(value) == 1 + isinstance(arr, np.ndarray) + and arr.ndim == 1 + and len(arr) == 1 ): # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 arr = arr[0] diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index f03f88306f8b2..5562884c6c4bb 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1063,9 +1063,6 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block: self = self.make_block_same_class( values.T if values.ndim == 2 else values ) - if isinstance(value, np.ndarray) and value.ndim == 1 and len(value) == 1: - # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 - casted = casted[0] values[indexer] = casted return self diff --git a/pandas/io/parsers/base_parser.py b/pandas/io/parsers/base_parser.py index 99fbfe46d22fc..0bac882756a91 100644 --- a/pandas/io/parsers/base_parser.py +++ b/pandas/io/parsers/base_parser.py @@ -1114,6 +1114,12 @@ def _make_date_converter( if date_parser is not lib.no_default and date_format is not None: raise TypeError("Cannot use both 'date_parser' and 'date_format'") + def unpack_if_single_element(arg): + # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 + if isinstance(arg, np.ndarray) and arg.ndim == 1 and len(arg) == 1: + return arg[0] + return arg + def converter(*date_cols, col: Hashable): if date_parser is lib.no_default: strs = parsing.concat_date_cols(date_cols) @@ -1137,7 +1143,9 @@ def converter(*date_cols, col: Hashable): else: try: result = tools.to_datetime( - date_parser(*date_cols), errors="ignore", cache=cache_dates + date_parser(*(unpack_if_single_element(arg) for arg in date_cols)), + errors="ignore", + cache=cache_dates, ) if isinstance(result, datetime.datetime): raise Exception("scalar parser") From 3ff2cba0c3f73c0b9a2e1bf7863dadead3bb6f37 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 25 Apr 2023 15:19:31 -0700 Subject: [PATCH 4/5] Undo whitespace --- pandas/core/internals/blocks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 5562884c6c4bb..ff022cb047f3d 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1063,6 +1063,7 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block: self = self.make_block_same_class( values.T if values.ndim == 2 else values ) + values[indexer] = casted return self From 771fe0f7f350b47fcd339d25dcc2feb0efb545d5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 25 Apr 2023 16:48:03 -0700 Subject: [PATCH 5/5] Turn into 0D array --- pandas/core/indexing.py | 2 +- pandas/core/internals/base.py | 2 +- pandas/core/internals/blocks.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index f97f92f4cb006..0f1cd397f2dd1 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1759,7 +1759,7 @@ def _setitem_with_indexer(self, indexer, value, name: str = "iloc"): and len(arr) == 1 ): # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 - arr = arr[0] + arr = arr[0, ...] empty_value[indexer[0]] = arr self.obj[key] = empty_value return diff --git a/pandas/core/internals/base.py b/pandas/core/internals/base.py index 922b3caf87cb9..8f771221c8890 100644 --- a/pandas/core/internals/base.py +++ b/pandas/core/internals/base.py @@ -189,7 +189,7 @@ def setitem_inplace(self, indexer, value) -> None: if isinstance(value, np.ndarray) and value.ndim == 1 and len(value) == 1: # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 - value = value[0] + value = value[0, ...] arr[indexer] = value diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index ff022cb047f3d..94670fe47036b 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1063,7 +1063,9 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block: self = self.make_block_same_class( values.T if values.ndim == 2 else values ) - + if isinstance(casted, np.ndarray) and casted.ndim == 1 and len(casted) == 1: + # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615 + casted = casted[0, ...] values[indexer] = casted return self