Remove --ci option

And move meta tests to top-level meta_tests folder

Author: honno

.github/workflows/numpy.yml

@@ -28,4 +28,4 @@ jobs:
       env:
         ARRAY_API_TESTS_MODULE: numpy.array_api
       run: |
-        pytest -v -rxXfE --ci --skips-file numpy-skips.txt
+        pytest -v -rxXfE --skips-file numpy-skips.txt array_api_tests/

README.md

@@ -161,12 +161,6 @@ You can specify the API version to use when testing via the
 array module's `__array_api_version__` value, and if that attribute doesn't
 exist then we fallback to `"2021.12"`.
 
-#### CI flag
-
-Use the `--ci` flag to run only the primary and special cases tests. You can
-ignore the other test cases as they are redundant for the purposes of checking
-compliance.
-
 #### Data-dependent shapes
 
 Use the `--disable-data-dependent-shapes` flag to skip testing functions which have

array_api_tests/test_array_object.py

@@ -16,8 +16,6 @@
 from . import xp as _xp
 from .typing import DataType, Index, Param, Scalar, ScalarType, Shape
 
-pytestmark = pytest.mark.ci
-
 
 def scalar_objects(
     dtype: DataType, shape: Shape

array_api_tests/test_constants.py

@@ -7,8 +7,6 @@
 from . import xp
 from .typing import Array
 
-pytestmark = pytest.mark.ci
-
 
 def assert_scalar_float(name: str, c: Any):
     assert isinstance(c, SupportsFloat), f"{name}={c!r} does not look like a float"

array_api_tests/test_creation_functions.py

@@ -3,7 +3,6 @@
 from itertools import count
 from typing import Iterator, NamedTuple, Union
 
-import pytest
 from hypothesis import assume, given, note
 from hypothesis import strategies as st
 
@@ -15,8 +14,6 @@
 from . import xps
 from .typing import DataType, Scalar
 
-pytestmark = pytest.mark.ci
-
 
 class frange(NamedTuple):
     start: float

array_api_tests/test_data_type_functions.py

@@ -14,8 +14,6 @@
 from . import xp as _xp
 from .typing import DataType
 
-pytestmark = pytest.mark.ci
-
 
 # TODO: test with complex dtypes
 def non_complex_dtypes():

array_api_tests/test_fft.py

@@ -17,7 +17,6 @@
 from . import xp
 
 pytestmark = [
-    pytest.mark.ci,
     pytest.mark.xp_extension("fft"),
     pytest.mark.min_version("2022.12"),
 ]

array_api_tests/test_has_names.py

@@ -9,8 +9,6 @@
 from .stubs import (array_attributes, array_methods, category_to_funcs,
                     extension_to_funcs, EXTENSIONS)
 
-pytestmark = pytest.mark.ci
-
 has_name_params = []
 for ext, stubs in extension_to_funcs.items():
     for stub in stubs:

array_api_tests/test_indexing_functions.py

@@ -9,8 +9,6 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = pytest.mark.ci
-
 
 @pytest.mark.min_version("2022.12")
 @given(

array_api_tests/test_linalg.py

@@ -43,8 +43,6 @@
 from . import _array_module as xp
 from ._array_module import linalg
 
-pytestmark = pytest.mark.ci
-
 def assert_equal(x, y, msg_extra=None):
     extra = '' if not msg_extra else f' ({msg_extra})'
     if x.dtype in dh.all_float_dtypes:

array_api_tests/test_manipulation_functions.py

@@ -14,8 +14,6 @@
 from . import xps
 from .typing import Array, Shape
 
-pytestmark = pytest.mark.ci
-
 MAX_SIDE = hh.MAX_ARRAY_SIZE // 64
 MAX_DIMS = min(hh.MAX_ARRAY_SIZE // MAX_SIDE, 32)  # NumPy only supports up to 32 dims
 

array_api_tests/test_operators_and_elementwise_functions.py

@@ -21,8 +21,6 @@
 from . import xps
 from .typing import Array, DataType, Param, Scalar, ScalarType, Shape
 
-pytestmark = pytest.mark.ci
-
 
 def all_integer_dtypes() -> st.SearchStrategy[DataType]:
     """Returns a strategy for signed and unsigned integer dtype objects."""

array_api_tests/test_searching_functions.py

@@ -11,8 +11,6 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = pytest.mark.ci
-
 
 @given(
     x=hh.arrays(

array_api_tests/test_set_functions.py

@@ -13,7 +13,7 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = [pytest.mark.ci, pytest.mark.data_dependent_shapes]
+pytestmark = pytest.mark.data_dependent_shapes
 
 
 @given(hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)))

array_api_tests/test_signatures.py

@@ -33,8 +33,6 @@ def squeeze(x, /, axis):
 from . import xp
 from .stubs import array_methods, category_to_funcs, extension_to_funcs, name_to_func
 
-pytestmark = pytest.mark.ci
-
 ParameterKind = Literal[
     Parameter.POSITIONAL_ONLY,
     Parameter.VAR_POSITIONAL,

array_api_tests/test_sorting_functions.py

@@ -1,7 +1,6 @@
 import cmath
 from typing import Set
 
-import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 from hypothesis.control import assume
@@ -14,8 +13,6 @@
 from . import xps
 from .typing import Scalar, Shape
 
-pytestmark = pytest.mark.ci
-
 
 def assert_scalar_in_set(
     func_name: str,

array_api_tests/test_special_cases.py

@@ -35,8 +35,6 @@
 from . import xp, xps
 from .stubs import category_to_funcs
 
-pytestmark = pytest.mark.ci
-
 UnaryCheck = Callable[[float], bool]
 BinaryCheck = Callable[[float, float], bool]
 

array_api_tests/test_statistical_functions.py

@@ -15,8 +15,6 @@
 from ._array_module import _UndefinedStub
 from .typing import DataType
 
-pytestmark = pytest.mark.ci
-
 
 def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
     dtypes = [d2 for d1, d2 in dh.promotion_table if d1 == dtype]

array_api_tests/test_utility_functions.py

@@ -1,4 +1,3 @@
-import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 
@@ -9,8 +8,6 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = pytest.mark.ci
-
 
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)),

conftest.py

@@ -54,11 +54,6 @@ def pytest_addoption(parser):
         help="disable testing functions with output shapes dependent on input",
     )
     # CI
-    parser.addoption(
-        "--ci",
-        action="store_true",
-        help="run just the tests appropriate for CI",
-    )
     parser.addoption(
         "--skips-file",
         action="store",
@@ -78,7 +73,6 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers", "data_dependent_shapes: output shapes are dependent on inputs"
     )
-    config.addinivalue_line("markers", "ci: primary test")
     config.addinivalue_line(
         "markers",
         "min_version(api_version): run when greater or equal to api_version",
@@ -144,7 +138,6 @@ def pytest_collection_modifyitems(config, items):
 
     disabled_exts = config.getoption("--disable-extension")
     disabled_dds = config.getoption("--disable-data-dependent-shapes")
-    ci = config.getoption("--ci")
 
     for item in items:
         markers = list(item.iter_markers())
@@ -178,11 +171,6 @@ def pytest_collection_modifyitems(config, items):
                         mark.skip(reason="disabled via --disable-data-dependent-shapes")
                     )
                     break
-        # skip if test not appropriate for CI
-        if ci:
-            ci_mark = next((m for m in markers if m.name == "ci"), None)
-            if ci_mark is None:
-                item.add_marker(mark.skip(reason="disabled via --ci"))
         # skip if test is for greater api_version
         ver_mark = next((m for m in markers if m.name == "min_version"), None)
         if ver_mark is not None:

meta_tests/README.md

@@ -0,0 +1 @@
+Testing the utilities used in `array_api_tests/`

array_api_tests/meta/__init__.py renamed to meta_tests/__init__.py

@@ -1,5 +1,5 @@
-from .. import _array_module as xp
-from ..array_helpers import exactly_equal, notequal
+from array_api_tests import _array_module as xp
+from array_api_tests .array_helpers import exactly_equal, notequal
 
 # TODO: These meta-tests currently only work with NumPy
 

array_api_tests/meta/test_array_helpers.py renamed to meta_tests/test_array_helpers.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from .. import shape_helpers as sh
+from array_api_tests import shape_helpers as sh
 
 
 @pytest.mark.parametrize(

array_api_tests/meta/test_broadcasting.py renamed to meta_tests/test_broadcasting.py

@@ -1,6 +1,6 @@
 import pytest
 
-from ..dtype_helpers import EqualityMapping
+from array_api_tests .dtype_helpers import EqualityMapping
 
 
 def test_raises_on_distinct_eq_key():

array_api_tests/meta/test_equality_mapping.py renamed to meta_tests/test_equality_mapping.py

@@ -6,13 +6,13 @@
 from hypothesis import strategies as st
 from hypothesis.errors import Unsatisfiable
 
-from .. import _array_module as xp
-from .. import array_helpers as ah
-from .. import dtype_helpers as dh
-from .. import hypothesis_helpers as hh
-from .. import shape_helpers as sh
-from .. import xps
-from .._array_module import _UndefinedStub
+from array_api_tests import _array_module as xp
+from array_api_tests import array_helpers as ah
+from array_api_tests import dtype_helpers as dh
+from array_api_tests import hypothesis_helpers as hh
+from array_api_tests import shape_helpers as sh
+from array_api_tests import xps
+from array_api_tests ._array_module import _UndefinedStub
 
 UNDEFINED_DTYPES = any(isinstance(d, _UndefinedStub) for d in dh.all_dtypes)
 pytestmark = [pytest.mark.skipif(UNDEFINED_DTYPES, reason="undefined dtypes")]

array_api_tests/meta/test_hypothesis_helpers.py renamed to meta_tests/test_hypothesis_helpers.py

@@ -2,9 +2,9 @@
 
 from hypothesis import given
 
-from ..hypothesis_helpers import symmetric_matrices
-from .. import array_helpers as ah
-from .. import _array_module as xp
+from array_api_tests .hypothesis_helpers import symmetric_matrices
+from array_api_tests import array_helpers as ah
+from array_api_tests import _array_module as xp
 
 @pytest.mark.xp_extension('linalg')
 @given(x=symmetric_matrices(finite=True))

array_api_tests/meta/test_linalg.py renamed to meta_tests/test_linalg.py

@@ -1,10 +1,10 @@
 import pytest
 from hypothesis import given
 
-from .. import dtype_helpers as dh
-from .. import hypothesis_helpers as hh
-from .. import _array_module as xp
-from .._array_module import _UndefinedStub
+from array_api_tests import dtype_helpers as dh
+from array_api_tests import hypothesis_helpers as hh
+from array_api_tests import _array_module as xp
+from array_api_tests ._array_module import _UndefinedStub
 
 
 # e.g. PyTorch only supports uint8 currently

array_api_tests/meta/test_partial_adopters.py renamed to meta_tests/test_partial_adopters.py

@@ -1,7 +1,7 @@
 from pytest import raises
 
-from .. import _array_module as xp
-from .. import pytest_helpers as ph
+from array_api_tests import _array_module as xp
+from array_api_tests import pytest_helpers as ph
 
 
 def test_assert_dtype():

array_api_tests/meta/test_pytest_helpers.py renamed to meta_tests/test_pytest_helpers.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from ..test_signatures import _test_inspectable_func
+from array_api_tests .test_signatures import _test_inspectable_func
 
 
 def stub(foo, /, bar=None, *, baz=None):

array_api_tests/meta/test_signatures.py renamed to meta_tests/test_signatures.py

@@ -1,6 +1,6 @@
 import math
 
-from ..test_special_cases import parse_result
+from array_api_tests .test_special_cases import parse_result
 
 
 def test_parse_result():

array_api_tests/meta/test_special_cases.py renamed to meta_tests/test_special_cases.py

@@ -2,14 +2,14 @@
 from hypothesis import given
 from hypothesis import strategies as st
 
-from .. import _array_module as xp
-from .. import dtype_helpers as dh
-from .. import hypothesis_helpers as hh
-from .. import shape_helpers as sh
-from .. import xps
-from ..test_creation_functions import frange
-from ..test_manipulation_functions import roll_ndindex
-from ..test_operators_and_elementwise_functions import mock_int_dtype
+from array_api_tests import _array_module as xp
+from array_api_tests import dtype_helpers as dh
+from array_api_tests import hypothesis_helpers as hh
+from array_api_tests import shape_helpers as sh
+from array_api_tests import xps
+from array_api_tests .test_creation_functions import frange
+from array_api_tests .test_manipulation_functions import roll_ndindex
+from array_api_tests .test_operators_and_elementwise_functions import mock_int_dtype
 
 
 @pytest.mark.parametrize(