Merge branch 'master' into 2023-fixes

Author: asmeurer

README.md

@@ -3,8 +3,8 @@
 This is the test suite for array libraries adopting the [Python Array API
 standard](https://data-apis.org/array-api/latest).
 
-Note the suite is still a **work in progress**. Feedback and contributions are
-welcome!
+Keeping full coverage of the spec is an on-going priority as the Array API evolves. 
+Feedback and contributions are welcome!
 
 ## Quickstart
 
@@ -33,11 +33,23 @@ You need to specify the array library to test. It can be specified via the
 `ARRAY_API_TESTS_MODULE` environment variable, e.g.
 
 ```bash
-$ export ARRAY_API_TESTS_MODULE=numpy.array_api
+$ export ARRAY_API_TESTS_MODULE=array_api_strict
 ```
 
 Alternately, import/define the `xp` variable in `array_api_tests/__init__.py`.
 
+### Specifying the API version
+
+You can specify the API version to use when testing via the
+`ARRAY_API_TESTS_VERSION` environment variable, e.g.
+
+```bash
+$ export ARRAY_API_TESTS_VERSION="2023.12"
+```
+
+Currently this defaults to the array module's `__array_api_version__` value, and
+if that attribute doesn't exist then we fallback to `"2021.12"`.
+
 ### Run the suite
 
 Simply run `pytest` against the `array_api_tests/` folder to run the full suite.
@@ -154,13 +166,6 @@ library to fail.
 
 ### Configuration
 
-#### API version
-
-You can specify the API version to use when testing via the
-`ARRAY_API_TESTS_VERSION` environment variable. Currently this defaults to the
-array module's `__array_api_version__` value, and if that attribute doesn't
-exist then we fallback to `"2021.12"`.
-
 #### Data-dependent shapes
 
 Use the `--disable-data-dependent-shapes` flag to skip testing functions which have
@@ -349,26 +354,6 @@ into a release. If you want, you can add release notes, which GitHub can
 generate for you.
 
 
-## Future plans
-
-Keeping full coverage of the spec is an on-going priority as the Array API
-evolves.
-
-Additionally, we have features and general improvements planned. Work on such
-functionality is guided primarily by the concerete needs of developers
-implementing and using the Array API—be sure to [let us
-know](https://github.com/data-apis/array-api-tests/issues) any limitations you
-come across.
-
-* A dependency graph for every test case, which could be used to modify pytest's
-  collection so that low-dependency tests are run first, and tests with faulty
-  dependencies would skip/xfail.
-
-* In some tests we've found it difficult to find appropaite assertion parameters
-  for output values (particularly epsilons for floating-point outputs), so we
-  need to review these and either implement assertions or properly note the lack
-  thereof.
-
 ---
 
 <sup>1</sup>The only exceptions to having just one primary test per function are:

array_api_tests/stubs.py

@@ -44,7 +44,7 @@
 
 category_to_funcs: Dict[str, List[FunctionType]] = {}
 for name, mod in name_to_mod.items():
-    if name.endswith("_functions"):
+    if name.endswith("_functions") or name == "info":  # info functions file just named info.py
         category = name.replace("_functions", "")
         objects = [getattr(mod, name) for name in mod.__all__]
         assert all(isinstance(o, FunctionType) for o in objects)  # sanity check

array_api_tests/test_linalg.py

@@ -37,6 +37,7 @@
 from . import dtype_helpers as dh
 from . import pytest_helpers as ph
 from . import shape_helpers as sh
+from . import api_version
 from .typing import Array
 
 from . import _array_module
@@ -873,7 +874,7 @@ def test_trace(x, kw):
         # See https://github.com/data-apis/array-api-tests/issues/160
         if x.dtype in dh.uint_dtypes:
             assert dh.is_int_dtype(res.dtype)  # sanity check
-    else:
+    elif api_version < "2023.12":  # TODO: update dtype assertion for >2023.12 - see #234
         ph.assert_dtype("trace", in_dtype=x.dtype, out_dtype=res.dtype, expected=expected_dtype)
 
     n, m = x.shape[-2:]

array_api_tests/test_statistical_functions.py

@@ -11,7 +11,7 @@
 from . import hypothesis_helpers as hh
 from . import pytest_helpers as ph
 from . import shape_helpers as sh
-from . import xps
+from . import api_version, xps
 from ._array_module import _UndefinedStub
 from .typing import DataType
 
@@ -148,7 +148,7 @@ def test_prod(x, data):
         # See https://github.com/data-apis/array-api-tests/issues/106
         if x.dtype in dh.uint_dtypes:
             assert dh.is_int_dtype(out.dtype)  # sanity check
-    else:
+    elif api_version < "2023.12":  # TODO: update dtype assertion for >2023.12 - see #234
         ph.assert_dtype("prod", in_dtype=x.dtype, out_dtype=out.dtype, expected=expected_dtype)
     _axes = sh.normalise_axis(kw.get("axis", None), x.ndim)
     ph.assert_keepdimable_shape(
@@ -207,7 +207,6 @@ def test_std(x, data):
 
 
 @pytest.mark.unvectorized
-@pytest.mark.skip("flaky")  # TODO: fix!
 @given(
     x=hh.arrays(
         dtype=xps.numeric_dtypes(),
@@ -238,7 +237,7 @@ def test_sum(x, data):
         # See https://github.com/data-apis/array-api-tests/issues/160
         if x.dtype in dh.uint_dtypes:
             assert dh.is_int_dtype(out.dtype)  # sanity check
-    else:
+    elif api_version < "2023.12":  # TODO: update dtype assertion for >2023.12 - see #234
         ph.assert_dtype("sum", in_dtype=x.dtype, out_dtype=out.dtype, expected=expected_dtype)
     _axes = sh.normalise_axis(kw.get("axis", None), x.ndim)
     ph.assert_keepdimable_shape(

reporting.py

@@ -103,7 +103,7 @@ def pytest_json_modifyreport(json_report):
     # doesn't store a full stack of where it was issued from. The resulting
     # warnings will be in order of the first time each warning is issued since
     # collections.Counter is ordered just like dict().
-    counted_warnings = Counter([frozenset(i.items()) for i in json_report['warnings']])
+    counted_warnings = Counter([frozenset(i.items()) for i in json_report.get('warnings', dict())])
     deduped_warnings = [{**dict(i), 'count': counted_warnings[i]} for i in counted_warnings]
 
     json_report['warnings'] = deduped_warnings