From 17b3eabe272d221426b4fd143afb9aa64d4a546a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 24 Jun 2022 16:13:42 -0600 Subject: [PATCH 01/23] Reintroduce test_has_names This just tests which names exist without checking anything else. It also doesn't use hypothesis, so it should work even if the library doesn't have proper hypothesis support (assuming the test suite can even run at all in that case, I'm not completely sure). --- array_api_tests/stubs.py | 4 ++++ array_api_tests/test_signatures.py | 35 +++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/array_api_tests/stubs.py b/array_api_tests/stubs.py index 35cc885f..04236edc 100644 --- a/array_api_tests/stubs.py +++ b/array_api_tests/stubs.py @@ -34,6 +34,10 @@ f for n, f in inspect.getmembers(array, predicate=inspect.isfunction) if n != "__init__" # probably exists for Sphinx ] +array_attributes = [ + n for n, f in inspect.getmembers(array, predicate=lambda x: not inspect.isfunction(x)) + if n != "__init__" # probably exists for Sphinx +] category_to_funcs: Dict[str, List[FunctionType]] = {} for name, mod in name_to_mod.items(): diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 2db804b1..9589c077 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -32,9 +32,9 @@ def squeeze(x, /, axis): from . import dtype_helpers as dh from . import hypothesis_helpers as hh from . import xps -from ._array_module import _UndefinedStub -from ._array_module import mod as xp -from .stubs import array_methods, category_to_funcs, extension_to_funcs +from ._array_module import _UndefinedStub, mod as xp, mod_name +from .stubs import (array_attributes, array_methods, category_to_funcs, + extension_to_funcs, EXTENSIONS) from .typing import Array, DataType pytestmark = pytest.mark.ci @@ -252,3 +252,32 @@ def test_array_method_signature(stub: FunctionType, data: DataObject): assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" method = getattr(x, stub.__name__) _test_func_signature(method, stub, array=x) + +has_name_params = [] +for ext, stubs in extension_to_funcs.items(): + for stub in stubs: + has_name_params.append(pytest.param(ext, stub.__name__)) +for cat, stubs in category_to_funcs.items(): + for stub in stubs: + has_name_params.append(pytest.param(cat, stub.__name__)) +for meth in array_methods: + has_name_params.append(pytest.param('array_method', meth.__name__)) +for attr in array_attributes: + has_name_params.append(pytest.param('array_attribute', attr)) + +# This is a very basic test to see what names are defined in a library. It +# does not even require functioning hypothesis array_api support. +@pytest.mark.parametrize("category, name", has_name_params) +def test_has_names(category, name): + if category in EXTENSIONS: + ext_mod = getattr(xp, category) + assert hasattr(ext_mod, name), f"{mod_name} is missing the {category} extension function {name}()" + elif category.startswith('array_'): + # TODO: This would fail if ones() is missing. + arr = xp.ones((1, 1)) + if category == 'array_attribute': + assert hasattr(arr, name), f"The {mod_name} array object is missing the attribute {name}" + else: + assert hasattr(arr, name), f"The {mod_name} array object is missing the method {name}()" + else: + assert hasattr(xp, name), f"{mod_name} is missing the {category} function {name}()" From fdb7ffdc5adb83103076b7229631014565319fde Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 24 Jun 2022 16:17:00 -0600 Subject: [PATCH 02/23] Move test_has_names to its own file That way it can be run even on libraries that fail the hypothesis sanity checks. --- array_api_tests/test_has_names.py | 37 ++++++++++++++++++++++++++++++ array_api_tests/test_signatures.py | 34 ++------------------------- 2 files changed, 39 insertions(+), 32 deletions(-) create mode 100644 array_api_tests/test_has_names.py diff --git a/array_api_tests/test_has_names.py b/array_api_tests/test_has_names.py new file mode 100644 index 00000000..d9194d82 --- /dev/null +++ b/array_api_tests/test_has_names.py @@ -0,0 +1,37 @@ +""" +This is a very basic test to see what names are defined in a library. It +does not even require functioning hypothesis array_api support. +""" + +import pytest + +from ._array_module import mod as xp, mod_name +from .stubs import (array_attributes, array_methods, category_to_funcs, + extension_to_funcs, EXTENSIONS) + +has_name_params = [] +for ext, stubs in extension_to_funcs.items(): + for stub in stubs: + has_name_params.append(pytest.param(ext, stub.__name__)) +for cat, stubs in category_to_funcs.items(): + for stub in stubs: + has_name_params.append(pytest.param(cat, stub.__name__)) +for meth in array_methods: + has_name_params.append(pytest.param('array_method', meth.__name__)) +for attr in array_attributes: + has_name_params.append(pytest.param('array_attribute', attr)) + +@pytest.mark.parametrize("category, name", has_name_params) +def test_has_names(category, name): + if category in EXTENSIONS: + ext_mod = getattr(xp, category) + assert hasattr(ext_mod, name), f"{mod_name} is missing the {category} extension function {name}()" + elif category.startswith('array_'): + # TODO: This would fail if ones() is missing. + arr = xp.ones((1, 1)) + if category == 'array_attribute': + assert hasattr(arr, name), f"The {mod_name} array object is missing the attribute {name}" + else: + assert hasattr(arr, name), f"The {mod_name} array object is missing the method {name}()" + else: + assert hasattr(xp, name), f"{mod_name} is missing the {category} function {name}()" diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index 9589c077..dd9d1255 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -32,9 +32,8 @@ def squeeze(x, /, axis): from . import dtype_helpers as dh from . import hypothesis_helpers as hh from . import xps -from ._array_module import _UndefinedStub, mod as xp, mod_name -from .stubs import (array_attributes, array_methods, category_to_funcs, - extension_to_funcs, EXTENSIONS) +from ._array_module import _UndefinedStub, mod as xp +from .stubs import array_methods, category_to_funcs, extension_to_funcs from .typing import Array, DataType pytestmark = pytest.mark.ci @@ -252,32 +251,3 @@ def test_array_method_signature(stub: FunctionType, data: DataObject): assert hasattr(x, stub.__name__), f"{stub.__name__} not found in array object {x!r}" method = getattr(x, stub.__name__) _test_func_signature(method, stub, array=x) - -has_name_params = [] -for ext, stubs in extension_to_funcs.items(): - for stub in stubs: - has_name_params.append(pytest.param(ext, stub.__name__)) -for cat, stubs in category_to_funcs.items(): - for stub in stubs: - has_name_params.append(pytest.param(cat, stub.__name__)) -for meth in array_methods: - has_name_params.append(pytest.param('array_method', meth.__name__)) -for attr in array_attributes: - has_name_params.append(pytest.param('array_attribute', attr)) - -# This is a very basic test to see what names are defined in a library. It -# does not even require functioning hypothesis array_api support. -@pytest.mark.parametrize("category, name", has_name_params) -def test_has_names(category, name): - if category in EXTENSIONS: - ext_mod = getattr(xp, category) - assert hasattr(ext_mod, name), f"{mod_name} is missing the {category} extension function {name}()" - elif category.startswith('array_'): - # TODO: This would fail if ones() is missing. - arr = xp.ones((1, 1)) - if category == 'array_attribute': - assert hasattr(arr, name), f"The {mod_name} array object is missing the attribute {name}" - else: - assert hasattr(arr, name), f"The {mod_name} array object is missing the method {name}()" - else: - assert hasattr(xp, name), f"{mod_name} is missing the {category} function {name}()" From cfbef56484afbd723e530ffd8182888add9279b8 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 24 Jun 2022 16:18:08 -0600 Subject: [PATCH 03/23] Add .report.json to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b6e47617..49d7dca5 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# pytest-json-report +.report.json From 7623741b6c25856b16a611a14ce4a58c21a08459 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 24 Jun 2022 16:18:29 -0600 Subject: [PATCH 04/23] Store the module name in the test metadata --- conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 9fec536b..e732d4b7 100644 --- a/conftest.py +++ b/conftest.py @@ -120,8 +120,15 @@ def pytest_collection_modifyitems(config, items): mark.skip(reason="disabled via --disable-data-dependent-shapes") ) break - # skip if test not appropiate for CI + # skip if test not appropriate for CI if ci: ci_mark = next((m for m in markers if m.name == "ci"), None) if ci_mark is None: item.add_marker(mark.skip(reason="disabled via --ci")) + +@mark.optionalhook +def pytest_metadata(metadata): + """ + Additional metadata for --json-report. + """ + metadata['array_api_tests_module'] = xp.mod_name From 3f25af0025c4940fc991079ccad6242b0a5df979 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 29 Jun 2022 17:43:00 -0600 Subject: [PATCH 05/23] Revert import change in test_signatures.py --- array_api_tests/test_signatures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/array_api_tests/test_signatures.py b/array_api_tests/test_signatures.py index dd9d1255..2db804b1 100644 --- a/array_api_tests/test_signatures.py +++ b/array_api_tests/test_signatures.py @@ -32,7 +32,8 @@ def squeeze(x, /, axis): from . import dtype_helpers as dh from . import hypothesis_helpers as hh from . import xps -from ._array_module import _UndefinedStub, mod as xp +from ._array_module import _UndefinedStub +from ._array_module import mod as xp from .stubs import array_methods, category_to_funcs, extension_to_funcs from .typing import Array, DataType From ed4d58e1364a877c40419ea98f3d5360426314ca Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 1 Aug 2022 13:16:20 -0600 Subject: [PATCH 06/23] Some work on adding additional metadata to the json report --- conftest.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index e732d4b7..27d6771a 100644 --- a/conftest.py +++ b/conftest.py @@ -2,7 +2,7 @@ from pathlib import Path from hypothesis import settings -from pytest import mark +from pytest import mark, fixture from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub @@ -132,3 +132,16 @@ def pytest_metadata(metadata): Additional metadata for --json-report. """ metadata['array_api_tests_module'] = xp.mod_name + +@fixture(autouse=True) +def add_api_name_to_metadata(request, json_metadata): + test_module = request.module.__name__ + test_function = request.function.__name__ + assert test_function.startswith('test_'), 'unexpected test function name' + + if test_module == 'array_api_tests.test_has_names': + array_api_function_name = None + else: + array_api_function_name = test_function[len('test_'):] + + json_metadata['array_api_function_name'] = array_api_function_name From 7f25ceca507d65da0327b86c49fde2f0dd9f87a2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 1 Aug 2022 19:04:38 -0600 Subject: [PATCH 07/23] Move reporting stuff to its own file, and add parametrize values to the report --- conftest.py | 25 +++------------------- reporting.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 22 deletions(-) create mode 100644 reporting.py diff --git a/conftest.py b/conftest.py index 27d6771a..f6852808 100644 --- a/conftest.py +++ b/conftest.py @@ -2,13 +2,14 @@ from pathlib import Path from hypothesis import settings -from pytest import mark, fixture +from pytest import mark from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub -settings.register_profile("xp_default", deadline=800) +from reporting import pytest_metadata, add_api_name_to_metadata # noqa +settings.register_profile("xp_default", deadline=800) def pytest_addoption(parser): # Hypothesis max examples @@ -125,23 +126,3 @@ def pytest_collection_modifyitems(config, items): ci_mark = next((m for m in markers if m.name == "ci"), None) if ci_mark is None: item.add_marker(mark.skip(reason="disabled via --ci")) - -@mark.optionalhook -def pytest_metadata(metadata): - """ - Additional metadata for --json-report. - """ - metadata['array_api_tests_module'] = xp.mod_name - -@fixture(autouse=True) -def add_api_name_to_metadata(request, json_metadata): - test_module = request.module.__name__ - test_function = request.function.__name__ - assert test_function.startswith('test_'), 'unexpected test function name' - - if test_module == 'array_api_tests.test_has_names': - array_api_function_name = None - else: - array_api_function_name = test_function[len('test_'):] - - json_metadata['array_api_function_name'] = array_api_function_name diff --git a/reporting.py b/reporting.py new file mode 100644 index 00000000..ad495709 --- /dev/null +++ b/reporting.py @@ -0,0 +1,58 @@ +from array_api_tests.test_operators_and_elementwise_functions import (UnaryParamContext, BinaryParamContext) +from array_api_tests.dtype_helpers import dtype_to_name +from array_api_tests import _array_module as xp + +from pytest import mark, fixture + +def to_json_serializable(o): + if o in dtype_to_name: + return dtype_to_name[o] + if isinstance(o, UnaryParamContext): + return {'func_name': o.func_name} + if isinstance(o, BinaryParamContext): + return { + 'func_name': o.func_name, + 'left_sym': o.left_sym, + 'right_sym': o.right_sym, + 'right_is_scalar': o.right_is_scalar, + 'res_name': o.res_name, + } + if isinstance(o, dict): + return {to_json_serializable(k): to_json_serializable(v) for k, v in o.items()} + if isinstance(o, tuple): + return tuple(to_json_serializable(i) for i in o) + if isinstance(o, list): + return [to_json_serializable(i) for i in o] + + return o + +@mark.optionalhook +def pytest_metadata(metadata): + """ + Additional global metadata for --json-report. + """ + metadata['array_api_tests_module'] = xp.mod_name + +@fixture(autouse=True) +def add_api_name_to_metadata(request, json_metadata): + """ + Additional per-test metadata for --json-report + """ + test_module = request.module.__name__ + if test_module.startswith('array_api_tests.meta'): + return + + test_function = request.function.__name__ + assert test_function.startswith('test_'), 'unexpected test function name' + + if test_module == 'array_api_tests.test_has_names': + array_api_function_name = None + else: + array_api_function_name = test_function[len('test_'):] + + json_metadata['test_module'] = test_module + json_metadata['test_function'] = test_function + json_metadata['array_api_function_name'] = array_api_function_name + + if hasattr(request.node, 'callspec'): + json_metadata['params'] = to_json_serializable(request.node.callspec.params) From 1f5284e42121c948ae9103cece240cd66afaf88c Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 1 Aug 2022 20:40:30 -0600 Subject: [PATCH 08/23] Make more types of test parameters JSON serializable --- reporting.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/reporting.py b/reporting.py index ad495709..664c9f46 100644 --- a/reporting.py +++ b/reporting.py @@ -1,25 +1,27 @@ -from array_api_tests.test_operators_and_elementwise_functions import (UnaryParamContext, BinaryParamContext) from array_api_tests.dtype_helpers import dtype_to_name from array_api_tests import _array_module as xp +from types import BuiltinFunctionType, FunctionType +import dataclasses + +from hypothesis.strategies import SearchStrategy + from pytest import mark, fixture def to_json_serializable(o): if o in dtype_to_name: return dtype_to_name[o] - if isinstance(o, UnaryParamContext): - return {'func_name': o.func_name} - if isinstance(o, BinaryParamContext): - return { - 'func_name': o.func_name, - 'left_sym': o.left_sym, - 'right_sym': o.right_sym, - 'right_is_scalar': o.right_is_scalar, - 'res_name': o.res_name, - } + if isinstance(o, (BuiltinFunctionType, FunctionType)): + return o.__name__ + if dataclasses.is_dataclass(o): + return to_json_serializable(dataclasses.asdict(o)) + if isinstance(o, SearchStrategy): + return repr(o) if isinstance(o, dict): return {to_json_serializable(k): to_json_serializable(v) for k, v in o.items()} if isinstance(o, tuple): + if hasattr(o, '_asdict'): # namedtuple + return to_json_serializable(o._asdict()) return tuple(to_json_serializable(i) for i in o) if isinstance(o, list): return [to_json_serializable(i) for i in o] @@ -55,4 +57,5 @@ def add_api_name_to_metadata(request, json_metadata): json_metadata['array_api_function_name'] = array_api_function_name if hasattr(request.node, 'callspec'): - json_metadata['params'] = to_json_serializable(request.node.callspec.params) + params = request.node.callspec.params + json_metadata['params'] = to_json_serializable(params) From d094951e6abb79b4f2155faf003fbf967b1785a2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 15:27:58 -0600 Subject: [PATCH 09/23] Include the array_api_tests version in the JSON report metadata --- reporting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reporting.py b/reporting.py index 664c9f46..dc2a5fad 100644 --- a/reporting.py +++ b/reporting.py @@ -1,5 +1,6 @@ from array_api_tests.dtype_helpers import dtype_to_name from array_api_tests import _array_module as xp +from array_api_tests import __version__ from types import BuiltinFunctionType, FunctionType import dataclasses @@ -34,6 +35,7 @@ def pytest_metadata(metadata): Additional global metadata for --json-report. """ metadata['array_api_tests_module'] = xp.mod_name + metadata['array_api_tests_version'] = __version__ @fixture(autouse=True) def add_api_name_to_metadata(request, json_metadata): From 0c941747df3dd8e3b92b4a3166f4f5142dc3ee6d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 15:28:13 -0600 Subject: [PATCH 10/23] Add hypothesis information to the JSON report metadata --- reporting.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/reporting.py b/reporting.py index dc2a5fad..079e8752 100644 --- a/reporting.py +++ b/reporting.py @@ -61,3 +61,13 @@ def add_api_name_to_metadata(request, json_metadata): if hasattr(request.node, 'callspec'): params = request.node.callspec.params json_metadata['params'] = to_json_serializable(params) + + def finalizer(): + # TODO: This metadata is all in the form of error strings. It might be + # nice to extract the hypothesis failing inputs directly somehow. + if hasattr(request.node, 'hypothesis_report_information'): + json_metadata['hypothesis_report_information'] = request.node.hypothesis_report_information + if hasattr(request.node, 'hypothesis_statistics'): + json_metadata['hypothesis_statistics'] = request.node.hypothesis_statistics + + request.addfinalizer(finalizer) From befbd80260808439eca50cd4a2d7ec0361a33554 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 16:25:23 -0600 Subject: [PATCH 11/23] Add a check that the custom JSON report metadata is always JSON serializable Otherwise it just shows up as a warning in the test results, which might easily be missed. --- reporting.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/reporting.py b/reporting.py index 079e8752..028968b5 100644 --- a/reporting.py +++ b/reporting.py @@ -4,6 +4,7 @@ from types import BuiltinFunctionType, FunctionType import dataclasses +import json from hypothesis.strategies import SearchStrategy @@ -42,6 +43,13 @@ def add_api_name_to_metadata(request, json_metadata): """ Additional per-test metadata for --json-report """ + def add_metadata(name, obj): + obj = to_json_serializable(obj) + # Ensure everything is JSON serializable. If this errors, it means the + # given type needs to be added to to_json_serializable above. + json.dumps(obj) + json_metadata[name] = obj + test_module = request.module.__name__ if test_module.startswith('array_api_tests.meta'): return @@ -54,20 +62,20 @@ def add_api_name_to_metadata(request, json_metadata): else: array_api_function_name = test_function[len('test_'):] - json_metadata['test_module'] = test_module - json_metadata['test_function'] = test_function - json_metadata['array_api_function_name'] = array_api_function_name + add_metadata('test_module', test_module) + add_metadata('test_function', test_function) + add_metadata('array_api_function_name', array_api_function_name) if hasattr(request.node, 'callspec'): params = request.node.callspec.params - json_metadata['params'] = to_json_serializable(params) + add_metadata('params', params) def finalizer(): # TODO: This metadata is all in the form of error strings. It might be # nice to extract the hypothesis failing inputs directly somehow. if hasattr(request.node, 'hypothesis_report_information'): - json_metadata['hypothesis_report_information'] = request.node.hypothesis_report_information + add_metadata('hypothesis_report_information', request.node.hypothesis_report_information) if hasattr(request.node, 'hypothesis_statistics'): - json_metadata['hypothesis_statistics'] = request.node.hypothesis_statistics + add_metadata('hypothesis_statistics', request.node.hypothesis_statistics) request.addfinalizer(finalizer) From 05c780257345c4c838cf15b83e9255363e975004 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 16:27:01 -0600 Subject: [PATCH 12/23] Add array_attributes to stubs.__all__ --- array_api_tests/stubs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/array_api_tests/stubs.py b/array_api_tests/stubs.py index 04236edc..3fba33fc 100644 --- a/array_api_tests/stubs.py +++ b/array_api_tests/stubs.py @@ -9,6 +9,7 @@ __all__ = [ "name_to_func", "array_methods", + "array_attributes", "category_to_funcs", "EXTENSIONS", "extension_to_funcs", From 9497ecd4efd37303e32a636250dbb33db8f24d6d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:20:45 -0600 Subject: [PATCH 13/23] Don't enable the add_api_name_to_metadata fixture unless --json-report is used --- conftest.py | 5 ++++- reporting.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index f6852808..233ac7af 100644 --- a/conftest.py +++ b/conftest.py @@ -2,7 +2,7 @@ from pathlib import Path from hypothesis import settings -from pytest import mark +from pytest import mark, fixture from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub @@ -126,3 +126,6 @@ def pytest_collection_modifyitems(config, items): ci_mark = next((m for m in markers if m.name == "ci"), None) if ci_mark is None: item.add_marker(mark.skip(reason="disabled via --ci")) + + if config.getoption('--json-report'): + fixture(autouse=True)(add_api_name_to_metadata) diff --git a/reporting.py b/reporting.py index 028968b5..61dbc5d5 100644 --- a/reporting.py +++ b/reporting.py @@ -38,7 +38,8 @@ def pytest_metadata(metadata): metadata['array_api_tests_module'] = xp.mod_name metadata['array_api_tests_version'] = __version__ -@fixture(autouse=True) +# This is dynamically decorated as a fixture in pytest_collection_modifyitems +# when --json-report is used. def add_api_name_to_metadata(request, json_metadata): """ Additional per-test metadata for --json-report From b5234ad69e6e86d95a581bfd417caa055a387117 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:21:31 -0600 Subject: [PATCH 14/23] Use a better name for the fixture --- reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting.py b/reporting.py index 61dbc5d5..95fab7b5 100644 --- a/reporting.py +++ b/reporting.py @@ -40,7 +40,7 @@ def pytest_metadata(metadata): # This is dynamically decorated as a fixture in pytest_collection_modifyitems # when --json-report is used. -def add_api_name_to_metadata(request, json_metadata): +def add_extra_json_metadata(request, json_metadata): """ Additional per-test metadata for --json-report """ From f8e562c0763483277fe50655f85fade89ddbccee Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:30:32 -0600 Subject: [PATCH 15/23] Fix undefined name --- conftest.py | 4 ++-- reporting.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index 233ac7af..8ab110e7 100644 --- a/conftest.py +++ b/conftest.py @@ -7,7 +7,7 @@ from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub -from reporting import pytest_metadata, add_api_name_to_metadata # noqa +from reporting import pytest_metadata, add_extra_json_metadata # noqa settings.register_profile("xp_default", deadline=800) @@ -128,4 +128,4 @@ def pytest_collection_modifyitems(config, items): item.add_marker(mark.skip(reason="disabled via --ci")) if config.getoption('--json-report'): - fixture(autouse=True)(add_api_name_to_metadata) + fixture(autouse=True)(add_extra_json_metadata) diff --git a/reporting.py b/reporting.py index 95fab7b5..0e55b78b 100644 --- a/reporting.py +++ b/reporting.py @@ -8,7 +8,7 @@ from hypothesis.strategies import SearchStrategy -from pytest import mark, fixture +from pytest import mark def to_json_serializable(o): if o in dtype_to_name: From 3dca2ad108ea0a50f2f8f45942772997ef32af9a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:35:17 -0600 Subject: [PATCH 16/23] Allow classes to be JSON serialized in the JSON report --- reporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting.py b/reporting.py index 0e55b78b..8dcbbeb1 100644 --- a/reporting.py +++ b/reporting.py @@ -13,7 +13,7 @@ def to_json_serializable(o): if o in dtype_to_name: return dtype_to_name[o] - if isinstance(o, (BuiltinFunctionType, FunctionType)): + if isinstance(o, (BuiltinFunctionType, FunctionType, type)): return o.__name__ if dataclasses.is_dataclass(o): return to_json_serializable(dataclasses.asdict(o)) From d62073a5244ae9445fb98a57c9a0a6d776cd37f2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:44:34 -0600 Subject: [PATCH 17/23] Add pytest-json-report to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b3b26223..de07fcc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ pytest +pytest-json-report hypothesis>=6.45.0 ndindex>=1.6 From 1c42d3a06a667aee387b26af8f0b1cabd2819178 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:46:49 -0600 Subject: [PATCH 18/23] Add a check that pytest-json-report is installed --- reporting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reporting.py b/reporting.py index 8dcbbeb1..2c2932de 100644 --- a/reporting.py +++ b/reporting.py @@ -9,6 +9,10 @@ from hypothesis.strategies import SearchStrategy from pytest import mark +try: + import pytest_jsonreport # noqa +except ImportError: + raise ImportError("pytest-json-report is required to run the array API tests") def to_json_serializable(o): if o in dtype_to_name: From 98db8935f7c44c51a1c6e8aaee8deb028c83a25b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:46:58 -0600 Subject: [PATCH 19/23] Remove attempt to only enable the fixture when --json-report is used It was actually not enabling it at all ever. --- conftest.py | 5 +---- reporting.py | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/conftest.py b/conftest.py index 8ab110e7..e982d3d4 100644 --- a/conftest.py +++ b/conftest.py @@ -2,7 +2,7 @@ from pathlib import Path from hypothesis import settings -from pytest import mark, fixture +from pytest import mark from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub @@ -126,6 +126,3 @@ def pytest_collection_modifyitems(config, items): ci_mark = next((m for m in markers if m.name == "ci"), None) if ci_mark is None: item.add_marker(mark.skip(reason="disabled via --ci")) - - if config.getoption('--json-report'): - fixture(autouse=True)(add_extra_json_metadata) diff --git a/reporting.py b/reporting.py index 2c2932de..04c3af4c 100644 --- a/reporting.py +++ b/reporting.py @@ -8,7 +8,7 @@ from hypothesis.strategies import SearchStrategy -from pytest import mark +from pytest import mark, fixture try: import pytest_jsonreport # noqa except ImportError: @@ -42,8 +42,7 @@ def pytest_metadata(metadata): metadata['array_api_tests_module'] = xp.mod_name metadata['array_api_tests_version'] = __version__ -# This is dynamically decorated as a fixture in pytest_collection_modifyitems -# when --json-report is used. +@fixture(autouse=True) def add_extra_json_metadata(request, json_metadata): """ Additional per-test metadata for --json-report From 775d25b5b00becea7aa15aedefe37d987544d465 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 Aug 2022 17:50:13 -0600 Subject: [PATCH 20/23] Add a repr() fallback for any non-JSON-serializable type The inputs could be things that aren't generically recognizable, e.g., for plain numpy, they could be numpy ufuncs. --- reporting.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/reporting.py b/reporting.py index 04c3af4c..f2c75158 100644 --- a/reporting.py +++ b/reporting.py @@ -5,6 +5,7 @@ from types import BuiltinFunctionType, FunctionType import dataclasses import json +import warnings from hypothesis.strategies import SearchStrategy @@ -32,6 +33,14 @@ def to_json_serializable(o): if isinstance(o, list): return [to_json_serializable(i) for i in o] + # Ensure everything is JSON serializable. If this warning is issued, it + # means the given type needs to be added above if possible. + try: + json.dumps(o) + except TypeError: + warnings.warn(f"{o!r} (of type {type(o)}) is not JSON-serializable. Using the repr instead.") + return repr(o) + return o @mark.optionalhook @@ -49,9 +58,6 @@ def add_extra_json_metadata(request, json_metadata): """ def add_metadata(name, obj): obj = to_json_serializable(obj) - # Ensure everything is JSON serializable. If this errors, it means the - # given type needs to be added to to_json_serializable above. - json.dumps(obj) json_metadata[name] = obj test_module = request.module.__name__ From f356ffd831c6b556c11b96295ecc10bb928f0ad5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 3 Aug 2022 14:04:54 -0600 Subject: [PATCH 21/23] Use shorter tracebacks in the JSON report --- conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conftest.py b/conftest.py index e982d3d4..80ee16cf 100644 --- a/conftest.py +++ b/conftest.py @@ -126,3 +126,8 @@ def pytest_collection_modifyitems(config, items): ci_mark = next((m for m in markers if m.name == "ci"), None) if ci_mark is None: item.add_marker(mark.skip(reason="disabled via --ci")) + + # Avoid long tracebacks for the JSON report, which make the file too + # large. + if config.getoption('--json-report'): + config.option.tbstyle = 'native' From a1ccd199e6ab2fc16c83d553e1d1ee9c5717846d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 3 Aug 2022 17:10:38 -0600 Subject: [PATCH 22/23] Revert "Use shorter tracebacks in the JSON report" This reverts commit f356ffd831c6b556c11b96295ecc10bb928f0ad5. It doesn't actually save that much space, so let's just use the default since it has more information and it keeps the code simpler. --- conftest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conftest.py b/conftest.py index 80ee16cf..e982d3d4 100644 --- a/conftest.py +++ b/conftest.py @@ -126,8 +126,3 @@ def pytest_collection_modifyitems(config, items): ci_mark = next((m for m in markers if m.name == "ci"), None) if ci_mark is None: item.add_marker(mark.skip(reason="disabled via --ci")) - - # Avoid long tracebacks for the JSON report, which make the file too - # large. - if config.getoption('--json-report'): - config.option.tbstyle = 'native' From 0841ef37c448e3491f319f2e16e25b6bb793351d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 4 Aug 2022 17:55:03 -0600 Subject: [PATCH 23/23] De-duplicate warnings metadata in the report JSON This was causing the report file sizes to be huge, especially for plain numpy. --- conftest.py | 2 +- reporting.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index e982d3d4..e0453e40 100644 --- a/conftest.py +++ b/conftest.py @@ -7,7 +7,7 @@ from array_api_tests import _array_module as xp from array_api_tests._array_module import _UndefinedStub -from reporting import pytest_metadata, add_extra_json_metadata # noqa +from reporting import pytest_metadata, pytest_json_modifyreport, add_extra_json_metadata # noqa settings.register_profile("xp_default", deadline=800) diff --git a/reporting.py b/reporting.py index f2c75158..b15b3364 100644 --- a/reporting.py +++ b/reporting.py @@ -2,6 +2,7 @@ from array_api_tests import _array_module as xp from array_api_tests import __version__ +from collections import Counter from types import BuiltinFunctionType, FunctionType import dataclasses import json @@ -89,3 +90,19 @@ def finalizer(): add_metadata('hypothesis_statistics', request.node.hypothesis_statistics) request.addfinalizer(finalizer) + +def pytest_json_modifyreport(json_report): + # Deduplicate warnings. These duplicate warnings can cause the file size + # to become huge. For instance, a warning from np.bool which is emitted + # every time hypothesis runs (over a million times) causes the warnings + # JSON for a plain numpy namespace run to be over 500MB. + + # This will lose information about what order the warnings were issued in, + # but that isn't particularly helpful anyway since the warning metadata + # doesn't store a full stack of where it was issued from. The resulting + # warnings will be in order of the first time each warning is issued since + # collections.Counter is ordered just like dict(). + counted_warnings = Counter([frozenset(i.items()) for i in json_report['warnings']]) + deduped_warnings = [{**dict(i), 'count': counted_warnings[i]} for i in counted_warnings] + + json_report['warnings'] = deduped_warnings