Skip to content

Commit 0f628db

Browse files
honnohonno
committed
Read xfails.txt for xfailing tests, update NumPy workflow
And merges both `conftest.py` files together
1 parent cb143f9 commit 0f628db

File tree

4 files changed

+96
-93
lines changed

4 files changed

+96
-93
lines changed

‎.github/workflows/numpy.yml

+27-31
Original file line numberDiff line numberDiff line change
@@ -22,39 +22,35 @@ jobs:
2222
python -m pip install git+https://github.com/numpy/numpy
2323
python -m pip install -r requirements.txt
2424
- name: Run the test suite
25+
env:
26+
ARRAY_API_TESTS_MODULE: numpy.array_api
2527
run: |
2628
# Mark some known issues as XFAIL
27-
cat << EOF >> conftest.py
29+
cat << EOF >> xfails.txt
30+
31+
# https://github.com/numpy/numpy/issues/18881
32+
test_creation_functions.py::test_linspace
33+
# einsum is not yet completed in the spec
34+
test_signatures.py::test_has_names[einsum]
35+
# dlpack support is not yet implemented in NumPy
36+
# See https://github.com/numpy/numpy/pull/19083
37+
test_signatures.py::test_function_positional_args[__dlpack__]
38+
test_signatures.py::test_function_positional_args[__dlpack_device__]
39+
test_signatures.py::test_function_positional_args[from_dlpack]
40+
test_signatures.py::test_function_keyword_only_args[__dlpack__]
41+
# Updates to the spec since the last change to numpy.array_api
42+
# These will fail until NumPy is updated
43+
test_signatures.py::test_has_names[__index__]
44+
test_signatures.py::test_has_names[to_device]
45+
test_signatures.py::test_has_names[mT]
46+
test_signatures.py::test_has_names[tril]
47+
test_signatures.py::test_has_names[triu]
48+
test_signatures.py::test_has_names[matrix_transpose]
49+
test_signatures.py::test_has_names[permute_dims]
50+
test_signatures.py::test_function_positional_args[__index__]
51+
test_signatures.py::test_function_keyword_only_args[prod]
52+
test_signatures.py::test_function_keyword_only_args[sum]
2853
29-
names_to_be_xfailed = (
30-
# https://github.com/numpy/numpy/issues/18881
31-
"array_api_tests/test_creation_functions.py::test_linspace",
32-
# einsum is not yet completed in the spec
33-
"array_api_tests/test_signatures.py::test_has_names[einsum]",
34-
# dlpack support is not yet implemented in NumPy. https://github.com/numpy/numpy/pull/19083
35-
"array_api_tests/test_signatures.py::test_function_positional_args[__dlpack__]",
36-
"array_api_tests/test_signatures.py::test_function_positional_args[__dlpack_device__]",
37-
"array_api_tests/test_signatures.py::test_function_positional_args[from_dlpack]",
38-
"array_api_tests/test_signatures.py::test_function_keyword_only_args[__dlpack__]",
39-
40-
# Updates to the spec since the last change to numpy.array_api.
41-
# These will fail until NumPy is updated.
42-
"array_api_tests/test_signatures.py::test_has_names[__index__]",
43-
"array_api_tests/test_signatures.py::test_has_names[to_device]",
44-
"array_api_tests/test_signatures.py::test_has_names[mT]",
45-
"array_api_tests/test_signatures.py::test_has_names[tril]",
46-
"array_api_tests/test_signatures.py::test_has_names[triu]",
47-
"array_api_tests/test_signatures.py::test_has_names[matrix_transpose]",
48-
"array_api_tests/test_signatures.py::test_has_names[permute_dims]",
49-
"array_api_tests/test_signatures.py::test_function_positional_args[__index__]",
50-
"array_api_tests/test_signatures.py::test_function_keyword_only_args[prod]",
51-
"array_api_tests/test_signatures.py::test_function_keyword_only_args[sum]",
52-
)
53-
54-
def pytest_collection_modifyitems(config, items):
55-
for item in items:
56-
if item.nodeid in names_to_be_xfailed:
57-
item.add_marker("xfail")
5854
EOF
5955
60-
ARRAY_API_TESTS_MODULE=numpy.array_api pytest -v -rxXfE
56+
pytest -v -rxXfE

‎array_api_tests/conftest.py

-57
This file was deleted.

‎array_api_tests/meta/test_utils.py

-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
from ..test_signatures import extension_module
2-
from ..conftest import xp_has_ext
32

43

54
def test_extension_module_is_extension():
@@ -8,7 +7,3 @@ def test_extension_module_is_extension():
87

98
def test_extension_func_is_not_extension():
109
assert not extension_module('linalg.cross')
11-
12-
13-
def test_xp_has_ext():
14-
assert not xp_has_ext('nonexistent_extension')

‎conftest.py

+69
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
1+
from functools import lru_cache
2+
from pathlib import Path
3+
4+
from pytest import mark
15
from hypothesis import settings
26

7+
from array_api_tests import _array_module as xp
8+
from array_api_tests._array_module import _UndefinedStub
9+
10+
311
settings.register_profile('xp_default', deadline=800)
412

513

@@ -20,9 +28,28 @@ def pytest_addoption(parser):
2028
action='store_true',
2129
help='disable the Hypothesis deadline',
2230
)
31+
# enable/disable extensions
32+
parser.addoption(
33+
'--enable-extension',
34+
metavar='ext',
35+
nargs='+',
36+
default=[],
37+
help='enable testing for Array API extension(s)',
38+
)
39+
parser.addoption(
40+
'--disable-extension',
41+
metavar='ext',
42+
nargs='+',
43+
default=[],
44+
help='disable testing for Array API extension(s)',
45+
)
2346

2447

2548
def pytest_configure(config):
49+
config.addinivalue_line(
50+
'markers', 'xp_extension(ext): tests an Array API extension'
51+
)
52+
# Hypothesis
2653
hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
2754
disable_deadline = config.getoption('--hypothesis-disable-deadline')
2855
profile_settings = {}
@@ -35,3 +62,45 @@ def pytest_configure(config):
3562
settings.load_profile('xp_override')
3663
else:
3764
settings.load_profile('xp_default')
65+
66+
67+
@lru_cache
68+
def xp_has_ext(ext: str) -> bool:
69+
try:
70+
return not isinstance(getattr(xp, ext), _UndefinedStub)
71+
except AttributeError:
72+
return False
73+
74+
75+
xfail_ids = []
76+
xfails_path = Path(__file__).parent / 'xfails.txt'
77+
if xfails_path.exists():
78+
with open(xfails_path) as f:
79+
for line in f:
80+
if line.startswith('test'):
81+
id_ = line.strip('\n')
82+
xfail_ids.append(f'array_api_tests/{id_}')
83+
84+
85+
def pytest_collection_modifyitems(config, items):
86+
enabled_exts = config.getoption('--enable-extension')
87+
disabled_exts = config.getoption('--disable-extension')
88+
for ext in enabled_exts:
89+
if ext in disabled_exts:
90+
raise ValueError(f'{ext=} both enabled and disabled')
91+
for item in items:
92+
# enable/disable extensions
93+
try:
94+
ext_mark = next(m for m in item.iter_markers() if m.name == 'xp_extension')
95+
ext = ext_mark.args[0]
96+
if ext in disabled_exts:
97+
item.add_marker(
98+
mark.skip(reason=f'{ext} disabled in --disable-extensions')
99+
)
100+
elif not ext in enabled_exts and not xp_has_ext(ext):
101+
item.add_marker(mark.skip(reason=f'{ext} not found in array module'))
102+
except StopIteration:
103+
pass
104+
# workflow xfail_ids
105+
if item.nodeid in xfail_ids:
106+
item.add_marker(mark.xfail(reason='xfails.txt'))

0 commit comments

Comments
 (0)