Add custom marker xp_extension()

Author: honno

conftest.py

@@ -1,45 +1,60 @@
 from hypothesis import settings
+from pytest import mark
+
+
+settings.register_profile('xp_default', deadline=800)
 
-# Add a --hypothesis-max-examples flag to pytest. See
-# https://github.com/HypothesisWorks/hypothesis/issues/2434#issuecomment-630309150
 
 def pytest_addoption(parser):
-    # Add an option to change the Hypothesis max_examples setting.
+    # Enable extensions
     parser.addoption(
-        "--hypothesis-max-examples",
-        "--max-examples",
-        action="store",
+        '--ext',
+        '--extensions',
+        nargs='+',
+        default=[],
+        help='enable testing for Array API extensions',
+    )
+    # Hypothesis max examples
+    # See https://github.com/HypothesisWorks/hypothesis/issues/2434
+    parser.addoption(
+        '--hypothesis-max-examples',
+        '--max-examples',
+        action='store',
         default=None,
-        help="set the Hypothesis max_examples setting",
+        help='set the Hypothesis max_examples setting',
     )
-
-    # Add an option to disable the Hypothesis deadline
+    # Hypothesis deadline
     parser.addoption(
-        "--hypothesis-disable-deadline",
-        "--disable-deadline",
-        action="store_true",
-        help="disable the Hypothesis deadline",
+        '--hypothesis-disable-deadline',
+        '--disable-deadline',
+        action='store_true',
+        help='disable the Hypothesis deadline',
     )
 
 
 def pytest_configure(config):
-    # Set Hypothesis max_examples.
-    hypothesis_max_examples = config.getoption("--hypothesis-max-examples")
+    config.addinivalue_line(
+        'markers', 'xp_extension(ext): tests an Array API extension'
+    )
+    # Configure Hypothesis
+    hypothesis_max_examples = config.getoption('--hypothesis-max-examples')
     disable_deadline = config.getoption('--hypothesis-disable-deadline')
     profile_settings = {}
     if hypothesis_max_examples is not None:
         profile_settings['max_examples'] = int(hypothesis_max_examples)
     if disable_deadline is not None:
         profile_settings['deadline'] = None
     if profile_settings:
-        import hypothesis
-
-        hypothesis.settings.register_profile(
-            "array-api-tests-hypothesis-overridden", **profile_settings,
-        )
-
-        hypothesis.settings.load_profile("array-api-tests-hypothesis-overridden")
-
-
-settings.register_profile('array_api_tests_hypothesis_profile', deadline=800)
-settings.load_profile('array_api_tests_hypothesis_profile')
+        settings.register_profile('xp_override', **profile_settings)
+        settings.load_profile('xp_override')
+    else:
+        settings.load_profile('xp_default')
+
+
+def pytest_collection_modifyitems(config, items):
+    exts = config.getoption('--extensions')
+    for item in items:
+        if 'xp_extension' in item.keywords:
+            ext = item.keywords['xp_extension'].args[0]
+            if ext not in exts:
+                item.add_marker(mark.skip(reason=f'{ext} not enabled in --extensions'))