Squashed commit of the following:

commit 3e01ab4
Author: Tom Augspurger <[email protected]>
Date:   Thu Mar 14 15:26:27 2019 -0500

    extra warnings option

commit ff66c6b
Author: Tom Augspurger <[email protected]>
Date:   Thu Mar 14 07:49:15 2019 -0500

    remove message

commit a55659c
Author: Tom Augspurger <[email protected]>
Date:   Thu Mar 14 06:55:49 2019 -0500

    handle no warning

commit c05306d
Author: Tom Augspurger <[email protected]>
Date:   Thu Mar 14 06:19:30 2019 -0500

    take 2

commit f0ad708
Author: Tom Augspurger <[email protected]>
Date:   Wed Mar 13 21:47:05 2019 -0500

    TST: assert_produces_warning works with filterwarnings

    Previously, assert_produces_warning did not play well with pytest's
    filterwarnings.

    ```
    ===================================================================================== FAILURES ======================================================================================
    ____________________________________________________________________ test_assert_produces_warning_honors_filter _____________________________________________________________________

        @pytest.mark.filterwarnings('ignore:f1:FutureWarning')
        def test_assert_produces_warning_honors_filter():
            with tm.assert_produces_warning(RuntimeWarning):
    >           f()

    pandas/tests/util/test_assert_produces_warning.py:17:
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

    self = <contextlib._GeneratorContextManager object at 0x11dd61128>, type = None, value = None, traceback = None

        def __exit__(self, type, value, traceback):
            if type is None:
                try:
    >               next(self.gen)
    E               AssertionError: Caused unexpected warning(s): [('FutureWarning', FutureWarning('f1'), '/Users/taugspurger/sandbox/pandas/pandas/tests/util/test_assert_produces_warni
    ng.py', 10)].

    /usr/local/Cellar/python/3.7.2_2/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py:119: AssertionError
    ============================================================================= 1 failed in 0.20 seconds ==============================================================================```

    Internally, `assert_produces_warning` sets a new `filter`, which
    overrides any filters set by pytest. We allow for `filter_level=None` to
    not set a filter.

Author: TomAugspurger

pandas/tests/util/test_assert_produces_warning.py

@@ -0,0 +1,22 @@
+import warnings
+
+import pytest
+
+import pandas.util.testing as tm
+
+
+def f():
+    warnings.warn('f1', FutureWarning)
+    warnings.warn('f2', RuntimeWarning)
+
+
+@pytest.mark.filterwarnings('ignore:f1:FutureWarning')
+def test_assert_produces_warning_honors_filter():
+    # raise by default
+    with pytest.raises(AssertionError):
+        with tm.assert_produces_warning(RuntimeWarning):
+            f()
+
+    with tm.assert_produces_warning(RuntimeWarning,
+                                    raise_on_extra_warnings=False):
+        f()

pandas/util/testing.py

@@ -2571,7 +2571,8 @@ def exception_matches(self, exc_type, exc_value, trace_back):
 
 @contextmanager
 def assert_produces_warning(expected_warning=Warning, filter_level="always",
-                            clear=None, check_stacklevel=True):
+                            clear=None, check_stacklevel=True,
+                            raise_on_extra_warnings=True):
     """
     Context manager for running code expected to either raise a specific
     warning, or not raise any warnings. Verifies that the code raises the
@@ -2584,7 +2585,7 @@ def assert_produces_warning(expected_warning=Warning, filter_level="always",
         The type of Exception raised. ``exception.Warning`` is the base
         class for all warnings. To check that no warning is returned,
         specify ``False`` or ``None``.
-    filter_level : str, default "always"
+    filter_level : str or None, default "always"
         Specifies whether warnings are ignored, displayed, or turned
         into errors.
         Valid values are:
@@ -2608,6 +2609,9 @@ class for all warnings. To check that no warning is returned,
         If True, displays the line that called the function containing
         the warning to show were the function is called. Otherwise, the
         line that implements the function is displayed.
+    raise_on_extra_warnings : bool, default True
+        Whether extra warnings not of the type `expected_warning` should
+        cause the test to fail.
 
     Examples
     --------
@@ -2676,8 +2680,10 @@ class for all warnings. To check that no warning is returned,
             msg = "Did not see expected warning of class {name!r}.".format(
                 name=expected_warning.__name__)
             assert saw_warning, msg
-        assert not extra_warnings, ("Caused unexpected warning(s): {extra!r}."
-                                    ).format(extra=extra_warnings)
+        if raise_on_extra_warnings and extra_warnings:
+            raise AssertionError(
+                "Caused unexpected warning(s): {!r}.".format(extra_warnings)
+            )
 
 
 class RNGContext(object):