From b91f849ca894bb076f9f1c0d6547d364b07d05a9 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 26 Apr 2020 22:17:23 -0700 Subject: [PATCH 1/3] ERR: Add NumbaUtilError --- .../reference/general_utility_functions.rst | 3 ++ pandas/core/groupby/generic.py | 3 -- pandas/core/util/numba_.py | 30 +++++-------------- pandas/errors/__init__.py | 6 ++++ pandas/tests/groupby/aggregate/test_numba.py | 9 +++--- pandas/tests/groupby/transform/test_numba.py | 9 +++--- pandas/tests/test_errors.py | 1 + 7 files changed, 28 insertions(+), 33 deletions(-) diff --git a/doc/source/reference/general_utility_functions.rst b/doc/source/reference/general_utility_functions.rst index 575b82b4b06de..6d43ceb74ab7c 100644 --- a/doc/source/reference/general_utility_functions.rst +++ b/doc/source/reference/general_utility_functions.rst @@ -35,9 +35,12 @@ Exceptions and warnings .. autosummary:: :toctree: api/ + errors.AccessorRegistrationWarning errors.DtypeWarning errors.EmptyDataError errors.OutOfBoundsDatetime + errors.MergeError + errors.NumbaUtilError errors.ParserError errors.ParserWarning errors.PerformanceWarning diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 18752cdc1642e..96ddd6eab24dd 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -79,7 +79,6 @@ NUMBA_FUNC_CACHE, check_kwargs_and_nopython, get_jit_arguments, - is_numba_util_related_error, jit_user_function, split_for_numba, validate_udf, @@ -285,8 +284,6 @@ def aggregate( ) except (ValueError, KeyError) as err: # Do not catch Numba errors here, we want to raise and not fall back. - if is_numba_util_related_error(str(err)): - raise err # TODO: KeyError is raised in _python_agg_general, # see see test_groupby.test_basic result = self._aggregate_named(func, *args, **kwargs) diff --git a/pandas/core/util/numba_.py b/pandas/core/util/numba_.py index 215248f5a43c2..c2e4b38ad5b4d 100644 --- a/pandas/core/util/numba_.py +++ b/pandas/core/util/numba_.py @@ -8,29 +8,11 @@ from pandas._typing import FrameOrSeries from pandas.compat._optional import import_optional_dependency +from pandas.errors import NumbaUtilError NUMBA_FUNC_CACHE: Dict[Tuple[Callable, str], Callable] = dict() -def is_numba_util_related_error(err_message: str) -> bool: - """ - Check if an error was raised from one of the numba utility functions - - For cases where a try/except block has mistakenly caught the error - and we want to re-raise - - Parameters - ---------- - err_message : str, - exception error message - - Returns - ------- - bool - """ - return "The first" in err_message or "numba does not" in err_message - - def check_kwargs_and_nopython( kwargs: Optional[Dict] = None, nopython: Optional[bool] = None ) -> None: @@ -51,10 +33,10 @@ def check_kwargs_and_nopython( Raises ------ - ValueError + NumbaUtilError """ if kwargs and nopython: - raise ValueError( + raise NumbaUtilError( "numba does not support kwargs with nopython=True: " "https://github.com/numba/numba/issues/2916" ) @@ -169,6 +151,10 @@ def f(values, index, ...): Returns ------- None + + Raises + ------ + NumbaUtilError """ udf_signature = list(inspect.signature(func).parameters.keys()) expected_args = ["values", "index"] @@ -177,7 +163,7 @@ def f(values, index, ...): len(udf_signature) < min_number_args or udf_signature[:min_number_args] != expected_args ): - raise ValueError( + raise NumbaUtilError( f"The first {min_number_args} arguments to {func.__name__} must be " f"{expected_args}" ) diff --git a/pandas/errors/__init__.py b/pandas/errors/__init__.py index 29e69cc5fe509..ef841d2dd4918 100644 --- a/pandas/errors/__init__.py +++ b/pandas/errors/__init__.py @@ -184,3 +184,9 @@ def __str__(self) -> str: else: name = type(self.class_instance).__name__ return f"This {self.methodtype} must be defined in the concrete class {name}" + + +class NumbaUtilError(Exception): + """ + Error raised for unsupported Numba engine routines. + """ diff --git a/pandas/tests/groupby/aggregate/test_numba.py b/pandas/tests/groupby/aggregate/test_numba.py index 70b0a027f1bd1..f23d7765eb508 100644 --- a/pandas/tests/groupby/aggregate/test_numba.py +++ b/pandas/tests/groupby/aggregate/test_numba.py @@ -1,6 +1,7 @@ import numpy as np import pytest +from pandas.errors import NumbaUtilError import pandas.util._test_decorators as td from pandas import DataFrame @@ -17,10 +18,10 @@ def incorrect_function(x): {"key": ["a", "a", "b", "b", "a"], "data": [1.0, 2.0, 3.0, 4.0, 5.0]}, columns=["key", "data"], ) - with pytest.raises(ValueError, match=f"The first 2"): + with pytest.raises(NumbaUtilError, match=f"The first 2"): data.groupby("key").agg(incorrect_function, engine="numba") - with pytest.raises(ValueError, match=f"The first 2"): + with pytest.raises(NumbaUtilError, match=f"The first 2"): data.groupby("key")["data"].agg(incorrect_function, engine="numba") @@ -33,10 +34,10 @@ def incorrect_function(x, **kwargs): {"key": ["a", "a", "b", "b", "a"], "data": [1.0, 2.0, 3.0, 4.0, 5.0]}, columns=["key", "data"], ) - with pytest.raises(ValueError, match="numba does not support"): + with pytest.raises(NumbaUtilError, match="numba does not support"): data.groupby("key").agg(incorrect_function, engine="numba", a=1) - with pytest.raises(ValueError, match="numba does not support"): + with pytest.raises(NumbaUtilError, match="numba does not support"): data.groupby("key")["data"].agg(incorrect_function, engine="numba", a=1) diff --git a/pandas/tests/groupby/transform/test_numba.py b/pandas/tests/groupby/transform/test_numba.py index 28904b669ae56..e2b957f1a7ae7 100644 --- a/pandas/tests/groupby/transform/test_numba.py +++ b/pandas/tests/groupby/transform/test_numba.py @@ -1,5 +1,6 @@ import pytest +from pandas.errors import NumbaUtilError import pandas.util._test_decorators as td from pandas import DataFrame @@ -16,10 +17,10 @@ def incorrect_function(x): {"key": ["a", "a", "b", "b", "a"], "data": [1.0, 2.0, 3.0, 4.0, 5.0]}, columns=["key", "data"], ) - with pytest.raises(ValueError, match=f"The first 2"): + with pytest.raises(NumbaUtilError, match=f"The first 2"): data.groupby("key").transform(incorrect_function, engine="numba") - with pytest.raises(ValueError, match=f"The first 2"): + with pytest.raises(NumbaUtilError, match=f"The first 2"): data.groupby("key")["data"].transform(incorrect_function, engine="numba") @@ -32,10 +33,10 @@ def incorrect_function(x, **kwargs): {"key": ["a", "a", "b", "b", "a"], "data": [1.0, 2.0, 3.0, 4.0, 5.0]}, columns=["key", "data"], ) - with pytest.raises(ValueError, match="numba does not support"): + with pytest.raises(NumbaUtilError, match="numba does not support"): data.groupby("key").transform(incorrect_function, engine="numba", a=1) - with pytest.raises(ValueError, match="numba does not support"): + with pytest.raises(NumbaUtilError, match="numba does not support"): data.groupby("key")["data"].transform(incorrect_function, engine="numba", a=1) diff --git a/pandas/tests/test_errors.py b/pandas/tests/test_errors.py index 515d798fe4322..6a1a74c73288f 100644 --- a/pandas/tests/test_errors.py +++ b/pandas/tests/test_errors.py @@ -18,6 +18,7 @@ "ParserWarning", "MergeError", "OptionError", + "NumbaUtilError", ], ) def test_exception_importable(exc): From 29975aa0d236958be9ca5c729c6317b251ffa02d Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 26 Apr 2020 23:17:21 -0700 Subject: [PATCH 2/3] Fix rolling test --- pandas/tests/window/test_apply.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/window/test_apply.py b/pandas/tests/window/test_apply.py index 7132e64c1191c..34cf0a3054889 100644 --- a/pandas/tests/window/test_apply.py +++ b/pandas/tests/window/test_apply.py @@ -1,6 +1,7 @@ import numpy as np import pytest +from pandas.errors import NumbaUtilError import pandas.util._test_decorators as td from pandas import DataFrame, Series, Timestamp, date_range @@ -134,7 +135,7 @@ def test_invalid_raw_numba(): @td.skip_if_no("numba") def test_invalid_kwargs_nopython(): - with pytest.raises(ValueError, match="numba does not support kwargs with"): + with pytest.raises(NumbaUtilError, match="numba does not support kwargs with"): Series(range(1)).rolling(1).apply( lambda x: x, kwargs={"a": 1}, engine="numba", raw=True ) From eaed1b42780e7bd2f888a7d1ab43f0782df59626 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 26 Apr 2020 23:40:12 -0700 Subject: [PATCH 3/3] lint --- pandas/core/groupby/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 96ddd6eab24dd..e90a4eae10853 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -282,7 +282,7 @@ def aggregate( return self._python_agg_general( func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs ) - except (ValueError, KeyError) as err: + except (ValueError, KeyError): # Do not catch Numba errors here, we want to raise and not fall back. # TODO: KeyError is raised in _python_agg_general, # see see test_groupby.test_basic