Merge pull request #4152 from HypothesisWorks/DRMacIver/ci-config

Default to CI-appropriate settings when running on CI

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: minor
+
+Hypothesis now detects if it is running on a CI server and provides better default settings for running on CI in this case.

hypothesis-python/docs/settings.rst

@@ -250,6 +250,14 @@ by your conftest you can load one with the command line option ``--hypothesis-pr
     $ pytest tests --hypothesis-profile <profile-name>
 
 
+Hypothesis comes with two built-in profiles, ``ci`` and ``default``.
+``ci`` is set up to have good defaults for running in a CI environment, so emphasizes determinism, while the 
+``default`` settings are picked to be more likely to find bugs and to have a good workflow when used for local development.
+
+Hypothesis will automatically detect certain common CI environments and use the CI profile automatically
+when running in them.
+In particular, if you wish to use the ``ci`` profile, setting the ``CI`` environment variable will do this.
+
 .. _healthchecks:
 
 -------------

hypothesis-python/src/hypothesis/_settings.py

@@ -63,6 +63,7 @@ def __get__(self, obj, type=None):
                     from hypothesis.database import ExampleDatabase
 
                     result = ExampleDatabase(not_set)
+                assert result is not not_set
                 return result
             except KeyError:
                 raise AttributeError(self.name) from None
@@ -407,6 +408,8 @@ def _max_examples_validator(x):
 :ref:`separate settings profiles <settings_profiles>` - for example running
 quick deterministic tests on every commit, and a longer non-deterministic
 nightly testing run.
+
+By default when running on CI, this will be set to True.
 """,
 )
 
@@ -682,6 +685,8 @@ def _validate_deadline(x):
 variability in test run time).
 
 Set this to ``None`` to disable this behaviour entirely.
+
+By default when running on CI, this will be set to None.
 """,
 )
 
@@ -694,13 +699,11 @@ def is_in_ci() -> bool:
 
 settings._define_setting(
     "print_blob",
-    default=is_in_ci(),
-    show_default=False,
+    default=False,
     options=(True, False),
     description="""
 If set to ``True``, Hypothesis will print code for failing examples that can be used with
 :func:`@reproduce_failure <hypothesis.reproduce_failure>` to reproduce the failing example.
-The default is ``True`` if the ``CI`` or ``TF_BUILD`` env vars are set, ``False`` otherwise.
 """,
 )
 
@@ -750,6 +753,24 @@ def note_deprecation(
 
 settings.register_profile("default", settings())
 settings.load_profile("default")
+
+assert settings.default is not None
+
+CI = settings(
+    derandomize=True,
+    deadline=None,
+    database=None,
+    print_blob=True,
+    suppress_health_check=[HealthCheck.too_slow],
+)
+
+settings.register_profile("ci", CI)
+
+
+# This is tested in a subprocess so the branch doesn't show up in coverage.
+if is_in_ci():  # pragma: no cover
+    settings.load_profile("ci")
+
 assert settings.default is not None
 
 

hypothesis-python/src/hypothesis/core.py

@@ -1564,23 +1564,23 @@ def wrapped_test(*arguments, **kwargs):
                     "to ensure that each example is run in a separate "
                     "database transaction."
                 )
-            if settings.database is not None:
-                nonlocal prev_self
-                # Check selfy really is self (not e.g. a mock) before we health-check
-                cur_self = (
-                    stuff.selfy
-                    if getattr(type(stuff.selfy), test.__name__, None) is wrapped_test
-                    else None
+
+            nonlocal prev_self
+            # Check selfy really is self (not e.g. a mock) before we health-check
+            cur_self = (
+                stuff.selfy
+                if getattr(type(stuff.selfy), test.__name__, None) is wrapped_test
+                else None
+            )
+            if prev_self is Unset:
+                prev_self = cur_self
+            elif cur_self is not prev_self:
+                msg = (
+                    f"The method {test.__qualname__} was called from multiple "
+                    "different executors. This may lead to flaky tests and "
+                    "nonreproducible errors when replaying from database."
                 )
-                if prev_self is Unset:
-                    prev_self = cur_self
-                elif cur_self is not prev_self:
-                    msg = (
-                        f"The method {test.__qualname__} was called from multiple "
-                        "different executors. This may lead to flaky tests and "
-                        "nonreproducible errors when replaying from database."
-                    )
-                    fail_health_check(settings, msg, HealthCheck.differing_executors)
+                fail_health_check(settings, msg, HealthCheck.differing_executors)
 
             state = StateForActualGivenExecution(
                 stuff, test, settings, random, wrapped_test
@@ -1675,7 +1675,6 @@ def wrapped_test(*arguments, **kwargs):
                 # The exception caught here should either be an actual test
                 # failure (or BaseExceptionGroup), or some kind of fatal error
                 # that caused the engine to stop.
-
                 generated_seed = wrapped_test._hypothesis_internal_use_generated_seed
                 with local_settings(settings):
                     if not (state.failed_normally or generated_seed is None):

hypothesis-python/tests/common/setup.py

@@ -12,7 +12,7 @@
 from warnings import filterwarnings
 
 from hypothesis import HealthCheck, Phase, Verbosity, settings
-from hypothesis._settings import not_set
+from hypothesis._settings import CI, is_in_ci, not_set
 from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 
@@ -45,13 +45,14 @@ def run():
         v = getattr(x, s.name)
         # Check if it has a dynamically defined default and if so skip comparison.
         if getattr(settings, s.name).show_default:
-            assert (
-                v == s.default
+            assert v == s.default or (
+                is_in_ci() and v == getattr(CI, s.name)
             ), f"({v!r} == x.{s.name}) != (s.{s.name} == {s.default!r})"
 
     settings.register_profile(
         "default",
         settings(
+            settings.get_profile("default"),
             max_examples=20 if IN_COVERAGE_TESTS else not_set,
             phases=list(Phase),  # Dogfooding the explain phase
         ),

hypothesis-python/tests/cover/test_settings.py

@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import datetime
+import os
 import subprocess
 import sys
 from unittest import TestCase
@@ -25,7 +26,7 @@
     note_deprecation,
     settings,
 )
-from hypothesis.database import ExampleDatabase
+from hypothesis.database import ExampleDatabase, InMemoryExampleDatabase
 from hypothesis.errors import (
     HypothesisDeprecationWarning,
     InvalidArgument,
@@ -108,12 +109,13 @@ def test_can_not_set_verbosity_to_non_verbosity():
 
 @pytest.mark.parametrize("db", [None, ExampleDatabase()])
 def test_inherits_an_empty_database(db):
-    assert settings.default.database is not None
-    s = settings(database=db)
-    assert s.database is db
-    with local_settings(s):
-        t = settings()
-    assert t.database is db
+    with local_settings(settings(database=InMemoryExampleDatabase())):
+        assert settings.default.database is not None
+        s = settings(database=db)
+        assert s.database is db
+        with local_settings(s):
+            t = settings()
+        assert t.database is db
 
 
 @pytest.mark.parametrize("db", [None, ExampleDatabase()])
@@ -273,6 +275,7 @@ def test_settings_as_decorator_must_be_on_callable():
 from hypothesis.configuration import set_hypothesis_home_dir
 from hypothesis.database import DirectoryBasedExampleDatabase
 
+settings.load_profile("default")
 settings.default.database
 
 if __name__ == '__main__':
@@ -476,8 +479,12 @@ def __repr__(self):
     assert "parent=(not settings repr)" in str(excinfo.value)
 
 
+def test_default_settings_do_not_use_ci():
+    assert settings.get_profile("default").suppress_health_check == ()
+
+
 def test_show_changed():
-    s = settings(max_examples=999, database=None)
+    s = settings(settings.get_profile("default"), max_examples=999, database=None)
     assert s.show_changed() == "database=None, max_examples=999"
 
 
@@ -511,3 +518,43 @@ def test_deprecated_settings_not_in_settings_all_list():
     assert al == ls
     assert HealthCheck.return_value not in ls
     assert HealthCheck.not_a_test_method not in ls
+
+
+@skipif_emscripten
+def test_check_defaults_to_derandomize_when_running_on_ci():
+    env = dict(os.environ)
+    env["CI"] = "true"
+
+    assert (
+        subprocess.check_output(
+            [
+                sys.executable,
+                "-c",
+                "from hypothesis import settings\nprint(settings().derandomize)",
+            ],
+            env=env,
+            text=True,
+            encoding="utf-8",
+        ).strip()
+        == "True"
+    )
+
+
+@skipif_emscripten
+def test_check_defaults_to_randomize_when_not_running_on_ci():
+    env = dict(os.environ)
+    env.pop("CI", None)
+    env.pop("TF_BUILD", None)
+    assert (
+        subprocess.check_output(
+            [
+                sys.executable,
+                "-c",
+                "from hypothesis import settings\nprint(settings().derandomize)",
+            ],
+            env=env,
+            text=True,
+            encoding="utf-8",
+        ).strip()
+        == "False"
+    )

hypothesis-python/tests/pytest/test_capture.py

@@ -93,7 +93,8 @@ def test_healthcheck_traceback_is_hidden(x):
 """
 
 
-def test_healthcheck_traceback_is_hidden(testdir):
+def test_healthcheck_traceback_is_hidden(testdir, monkeypatch):
+    monkeypatch.delenv("CI", raising=False)
     script = testdir.makepyfile(TRACEBACKHIDE_HEALTHCHECK)
     result = testdir.runpytest(script, "--verbose")
     def_token = "__ test_healthcheck_traceback_is_hidden __"

hypothesis-python/tests/pytest/test_seeding.py

@@ -82,7 +82,10 @@ def test_failure(i):
 """
 
 
-def test_repeats_healthcheck_when_following_seed_instruction(testdir, tmp_path):
+def test_repeats_healthcheck_when_following_seed_instruction(
+    testdir, tmp_path, monkeypatch
+):
+    monkeypatch.delenv("CI", raising=False)
     health_check_test = HEALTH_CHECK_FAILURE.replace(
         "<file>", repr(str(tmp_path / "seen"))
     )