diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 8f415b0932..2cbff44fc3 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -16,7 +16,7 @@ It also brings some dreadfully awaited fixes, so be sure to go through the chang
 - Removed `theanof.set_theano_config` because it illegally changed Theano's internal state (see [#4329](https://github.com/pymc-devs/pymc3/pull/4329)).
 
 ### New Features
-- Option to `disable_bounds_check=True` when instantiating `pymc3.Model()` for faster sampling for models that cannot violate boundary constraints (which are most of them; see [#4377](https://github.com/pymc-devs/pymc3/pull/4377)).
+- Option to set `check_bounds=False` when instantiating `pymc3.Model()`. This turns off bounds checks that ensure that input parameters of distributions are valid. For correctly specified models, this is unneccessary as all parameters get automatically transformed so that all values are valid. Turning this off should lead to faster sampling (see [#4377](https://github.com/pymc-devs/pymc3/pull/4377)).
 - `OrderedProbit` distribution added (see [#4232](https://github.com/pymc-devs/pymc3/pull/4232)).
 - `plot_posterior_predictive_glm` now works with `arviz.InferenceData` as well (see [#4234](https://github.com/pymc-devs/pymc3/pull/4234))
 
diff --git a/pymc3/distributions/dist_math.py b/pymc3/distributions/dist_math.py
index dc67f89652..12d5831330 100644
--- a/pymc3/distributions/dist_math.py
+++ b/pymc3/distributions/dist_math.py
@@ -72,7 +72,7 @@ def bound(logp, *conditions, **kwargs):
     # If called inside a model context, see if bounds check is disabled
     try:
         model = modelcontext(kwargs.get("model"))
-        if model.disable_bounds_check:
+        if not model.check_bounds:
             return logp
     except TypeError:  # No model found
         pass
diff --git a/pymc3/model.py b/pymc3/model.py
index 6a6e159b6c..793bbefc45 100644
--- a/pymc3/model.py
+++ b/pymc3/model.py
@@ -809,11 +809,11 @@ class Model(Factor, WithMemoization, metaclass=ContextMeta):
         temporarily in the model context. See the documentation
         of theano for a complete list. Set config key
         ``compute_test_value`` to `raise` if it is None.
-    disable_bounds_check: bool
-        Disable checks that ensure that input parameters to distributions
-        are in a valid range. If your model is built in a way where you
-        know your parameters can only take on valid values you can disable
-        this for increased speed.
+    check_bounds: bool
+        Ensure that input parameters to distributions are in a valid
+        range. If your model is built in a way where you know your
+        parameters can only take on valid values you can set this to
+        False for increased speed.
 
     Examples
     --------
@@ -900,14 +900,12 @@ def __new__(cls, *args, **kwargs):
         instance._theano_config = theano_config
         return instance
 
-    def __init__(
-        self, name="", model=None, theano_config=None, coords=None, disable_bounds_check=False
-    ):
+    def __init__(self, name="", model=None, theano_config=None, coords=None, check_bounds=True):
         self.name = name
         self.coords = {}
         self.RV_dims = {}
         self.add_coords(coords)
-        self.disable_bounds_check = disable_bounds_check
+        self.check_bounds = check_bounds
 
         if self.parent is not None:
             self.named_vars = treedict(parent=self.parent.named_vars)
diff --git a/pymc3/tests/test_dist_math.py b/pymc3/tests/test_dist_math.py
index 359cc8d656..2cf7b8e98f 100644
--- a/pymc3/tests/test_dist_math.py
+++ b/pymc3/tests/test_dist_math.py
@@ -60,8 +60,8 @@ def test_bound():
     assert np.prod(bound(logp, cond).eval()) == -np.inf
 
 
-def test_bound_disabled():
-    with pm.Model(disable_bounds_check=True):
+def test_check_bounds_false():
+    with pm.Model(check_bounds=False):
         logp = tt.ones(3)
         cond = np.array([1, 0, 1])
         assert np.all(bound(logp, cond).eval() == logp.eval())