diff --git a/ci/code_checks.sh b/ci/code_checks.sh
index 0b9aae6676710..f4761c5663c9f 100755
--- a/ci/code_checks.sh
+++ b/ci/code_checks.sh
@@ -208,7 +208,7 @@ import sys
 import pandas
 
 blacklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
-             'lxml', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
+             'lxml', 'matplotlib', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
              'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
 
 # GH#28227 for some of these check for top-level modules, while others are
diff --git a/doc/source/user_guide/visualization.rst b/doc/source/user_guide/visualization.rst
index fa16b2f216610..342e87289e993 100644
--- a/doc/source/user_guide/visualization.rst
+++ b/doc/source/user_guide/visualization.rst
@@ -1190,6 +1190,21 @@ with "(right)" in the legend. To turn off the automatic marking, use the
 
    plt.close('all')
 
+.. _plotting.formatters:
+
+Custom formatters for timeseries plots
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionchanged:: 1.0.0
+
+Pandas provides custom formatters for timeseries plots. These change the
+formatting of the axis labels for dates and times. By default,
+the custom formatters are applied only to plots created by pandas with
+:meth:`DataFrame.plot` or :meth:`Series.plot`. To have them apply to all
+plots, including those made by matplotlib, set the option
+``pd.options.plotting.matplotlib.register_converters = True`` or use
+:meth:`pandas.plotting.register_matplotlib_converters`.
+
 Suppressing tick resolution adjustment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst
index 48c1173a372a7..9262e8c325be3 100644
--- a/doc/source/whatsnew/v1.0.0.rst
+++ b/doc/source/whatsnew/v1.0.0.rst
@@ -174,7 +174,6 @@ Backwards incompatible API changes
 
    pd.arrays.IntervalArray.from_tuples([(0, 1), (2, 3)])
 
-
 .. _whatsnew_1000.api.other:
 
 Other API changes
@@ -186,8 +185,13 @@ Other API changes
 - In order to improve tab-completion, Pandas does not include most deprecated attributes when introspecting a pandas object using ``dir`` (e.g. ``dir(df)``).
   To see which attributes are excluded, see an object's ``_deprecations`` attribute, for example ``pd.DataFrame._deprecations`` (:issue:`28805`).
 - The returned dtype of ::func:`pd.unique` now matches the input dtype. (:issue:`27874`)
+- Changed the default configuration value for ``options.matplotlib.register_converters`` from ``True`` to ``"auto"`` (:issue:`18720`).
+  Now, pandas custom formatters will only be applied to plots created by pandas, through :meth:`~DataFrame.plot`.
+  Previously, pandas' formatters would be applied to all plots created *after* a :meth:`~DataFrame.plot`.
+  See :ref:`units registration <whatsnew_1000.matplotlib_units>` for more.
 -
 
+
 .. _whatsnew_1000.api.documentation:
 
 Documentation Improvements
@@ -220,6 +224,27 @@ with migrating existing code.
 Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. _whatsnew_1000.matplotlib_units:
+
+**Matplotlib unit registration**
+
+Previously, pandas would register converters with matplotlib as a side effect of importing pandas (:issue:`18720`).
+This changed the output of plots made via matplotlib plots after pandas was imported, even if you were using
+matplotlib directly rather than rather than :meth:`~DataFrame.plot`.
+
+To use pandas formatters with a matplotlib plot, specify
+
+.. code-block:: python
+
+   >>> import pandas as pd
+   >>> pd.options.plotting.matplotlib.register_converters = True
+
+Note that plots created by :meth:`DataFrame.plot` and :meth:`Series.plot` *do* register the converters
+automatically. The only behavior change is when plotting a date-like object via ``matplotlib.pyplot.plot``
+or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
+
+**Other removals**
+
 - Removed the previously deprecated :meth:`Series.get_value`, :meth:`Series.set_value`, :meth:`DataFrame.get_value`, :meth:`DataFrame.set_value` (:issue:`17739`)
 - Changed the the default value of `inplace` in :meth:`DataFrame.set_index` and :meth:`Series.set_axis`. It now defaults to False (:issue:`27600`)
 - :meth:`pandas.Series.str.cat` now defaults to aligning ``others``, using ``join='left'`` (:issue:`27611`)
diff --git a/pandas/core/config_init.py b/pandas/core/config_init.py
index bc2eb3511629d..ba0a4d81a88d3 100644
--- a/pandas/core/config_init.py
+++ b/pandas/core/config_init.py
@@ -599,7 +599,7 @@ def register_plotting_backend_cb(key):
 
 
 register_converter_doc = """
-: bool
+: bool or 'auto'.
     Whether to register converters with matplotlib's units registry for
     dates, times, datetimes, and Periods. Toggling to False will remove
     the converters, restoring any converters that pandas overwrote.
@@ -619,8 +619,8 @@ def register_converter_cb(key):
 with cf.config_prefix("plotting.matplotlib"):
     cf.register_option(
         "register_converters",
-        True,
+        "auto",
         register_converter_doc,
-        validator=bool,
+        validator=is_one_of_factory(["auto", True, False]),
         cb=register_converter_cb,
     )
diff --git a/pandas/plotting/_core.py b/pandas/plotting/_core.py
index eaf5b336bb8f6..127fdffafcf36 100644
--- a/pandas/plotting/_core.py
+++ b/pandas/plotting/_core.py
@@ -3,7 +3,6 @@
 
 from pandas._config import get_option
 
-from pandas.compat._optional import import_optional_dependency
 from pandas.util._decorators import Appender
 
 from pandas.core.dtypes.common import is_integer, is_list_like
@@ -11,11 +10,6 @@
 
 from pandas.core.base import PandasObject
 
-# Trigger matplotlib import, which implicitly registers our
-# converts. Implicit registration is deprecated, and when enforced
-# we can lazily import matplotlib.
-import_optional_dependency("pandas.plotting._matplotlib", raise_on_missing=False)
-
 
 def hist_series(
     self,
diff --git a/pandas/plotting/_matplotlib/__init__.py b/pandas/plotting/_matplotlib/__init__.py
index d3b7a34b6c923..206600ad37acc 100644
--- a/pandas/plotting/_matplotlib/__init__.py
+++ b/pandas/plotting/_matplotlib/__init__.py
@@ -1,5 +1,3 @@
-from pandas._config import get_option
-
 from pandas.plotting._matplotlib.boxplot import (
     BoxPlot,
     boxplot,
@@ -42,9 +40,6 @@
     "hexbin": HexBinPlot,
 }
 
-if get_option("plotting.matplotlib.register_converters"):
-    register(explicit=False)
-
 
 def plot(data, kind, **kwargs):
     # Importing pyplot at the top of the file (before the converters are
diff --git a/pandas/plotting/_matplotlib/boxplot.py b/pandas/plotting/_matplotlib/boxplot.py
index eed328131da92..cfd6c3519d82c 100644
--- a/pandas/plotting/_matplotlib/boxplot.py
+++ b/pandas/plotting/_matplotlib/boxplot.py
@@ -11,7 +11,6 @@
 import pandas as pd
 
 from pandas.io.formats.printing import pprint_thing
-from pandas.plotting._matplotlib import converter
 from pandas.plotting._matplotlib.core import LinePlot, MPLPlot
 from pandas.plotting._matplotlib.style import _get_standard_colors
 from pandas.plotting._matplotlib.tools import _flatten, _subplots
@@ -364,7 +363,6 @@ def boxplot_frame(
 ):
     import matplotlib.pyplot as plt
 
-    converter._WARN = False  # no warning for pandas plots
     ax = boxplot(
         self,
         column=column,
@@ -396,7 +394,6 @@ def boxplot_frame_groupby(
     sharey=True,
     **kwds
 ):
-    converter._WARN = False  # no warning for pandas plots
     if subplots is True:
         naxes = len(grouped)
         fig, axes = _subplots(
diff --git a/pandas/plotting/_matplotlib/converter.py b/pandas/plotting/_matplotlib/converter.py
index 446350cb5d915..946ce8bcec97f 100644
--- a/pandas/plotting/_matplotlib/converter.py
+++ b/pandas/plotting/_matplotlib/converter.py
@@ -1,6 +1,7 @@
+import contextlib
 import datetime as pydt
 from datetime import datetime, timedelta
-import warnings
+import functools
 
 from dateutil.relativedelta import relativedelta
 import matplotlib.dates as dates
@@ -23,6 +24,7 @@
 )
 from pandas.core.dtypes.generic import ABCSeries
 
+from pandas import get_option
 import pandas.core.common as com
 from pandas.core.index import Index
 from pandas.core.indexes.datetimes import date_range
@@ -39,7 +41,6 @@
 
 MUSEC_PER_DAY = 1e6 * SEC_PER_DAY
 
-_WARN = True  # Global for whether pandas has registered the units explicitly
 _mpl_units = {}  # Cache for units overwritten by us
 
 
@@ -55,13 +56,42 @@ def get_pairs():
     return pairs
 
 
-def register(explicit=True):
-    # Renamed in pandas.plotting.__init__
-    global _WARN
+def register_pandas_matplotlib_converters(func):
+    """
+    Decorator applying pandas_converters.
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        with pandas_converters():
+            return func(*args, **kwargs)
 
-    if explicit:
-        _WARN = False
+    return wrapper
 
+
+@contextlib.contextmanager
+def pandas_converters():
+    """
+    Context manager registering pandas' converters for a plot.
+
+    See Also
+    --------
+    register_pandas_matplotlib_converters : Decorator that applies this.
+    """
+    value = get_option("plotting.matplotlib.register_converters")
+
+    if value:
+        # register for True or "auto"
+        register()
+    try:
+        yield
+    finally:
+        if value == "auto":
+            # only deregister for "auto"
+            deregister()
+
+
+def register():
     pairs = get_pairs()
     for type_, cls in pairs:
         # Cache previous converter if present
@@ -86,24 +116,6 @@ def deregister():
             units.registry[unit] = formatter
 
 
-def _check_implicitly_registered():
-    global _WARN
-
-    if _WARN:
-        msg = (
-            "Using an implicitly registered datetime converter for a "
-            "matplotlib plotting method. The converter was registered "
-            "by pandas on import. Future versions of pandas will require "
-            "you to explicitly register matplotlib converters.\n\n"
-            "To register the converters:\n\t"
-            ">>> from pandas.plotting import register_matplotlib_converters"
-            "\n\t"
-            ">>> register_matplotlib_converters()"
-        )
-        warnings.warn(msg, FutureWarning)
-        _WARN = False
-
-
 def _to_ordinalf(tm):
     tot_sec = tm.hour * 3600 + tm.minute * 60 + tm.second + float(tm.microsecond / 1e6)
     return tot_sec
@@ -253,7 +265,6 @@ class DatetimeConverter(dates.DateConverter):
     @staticmethod
     def convert(values, unit, axis):
         # values might be a 1-d array, or a list-like of arrays.
-        _check_implicitly_registered()
         if is_nested_list_like(values):
             values = [DatetimeConverter._convert_1d(v, unit, axis) for v in values]
         else:
@@ -330,7 +341,6 @@ def __init__(self, locator, tz=None, defaultfmt="%Y-%m-%d"):
 class PandasAutoDateLocator(dates.AutoDateLocator):
     def get_locator(self, dmin, dmax):
         """Pick the best locator based on a distance."""
-        _check_implicitly_registered()
         delta = relativedelta(dmax, dmin)
 
         num_days = (delta.years * 12.0 + delta.months) * 31.0 + delta.days
@@ -372,7 +382,6 @@ def get_unit_generic(freq):
 
     def __call__(self):
         # if no data have been set, this will tank with a ValueError
-        _check_implicitly_registered()
         try:
             dmin, dmax = self.viewlim_to_dt()
         except ValueError:
@@ -990,7 +999,6 @@ def _get_default_locs(self, vmin, vmax):
     def __call__(self):
         "Return the locations of the ticks."
         # axis calls Locator.set_axis inside set_m<xxxx>_formatter
-        _check_implicitly_registered()
 
         vi = tuple(self.axis.get_view_interval())
         if vi != self.plot_obj.view_interval:
@@ -1075,7 +1083,6 @@ def set_locs(self, locs):
         "Sets the locations of the ticks"
         # don't actually use the locs. This is just needed to work with
         # matplotlib. Force to use vmin, vmax
-        _check_implicitly_registered()
 
         self.locs = locs
 
@@ -1088,7 +1095,6 @@ def set_locs(self, locs):
         self._set_default_format(vmin, vmax)
 
     def __call__(self, x, pos=0):
-        _check_implicitly_registered()
 
         if self.formatdict is None:
             return ""
@@ -1120,7 +1126,6 @@ def format_timedelta_ticks(x, pos, n_decimals):
         return s
 
     def __call__(self, x, pos=0):
-        _check_implicitly_registered()
         (vmin, vmax) = tuple(self.axis.get_view_interval())
         n_decimals = int(np.ceil(np.log10(100 * 1e9 / (vmax - vmin))))
         if n_decimals > 9:
diff --git a/pandas/plotting/_matplotlib/core.py b/pandas/plotting/_matplotlib/core.py
index a729951b3d7db..541dca715e814 100644
--- a/pandas/plotting/_matplotlib/core.py
+++ b/pandas/plotting/_matplotlib/core.py
@@ -4,8 +4,6 @@
 
 import numpy as np
 
-from pandas._config import get_option
-
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import cache_readonly
 
@@ -28,8 +26,8 @@
 import pandas.core.common as com
 
 from pandas.io.formats.printing import pprint_thing
-from pandas.plotting._matplotlib import converter
 from pandas.plotting._matplotlib.compat import _mpl_ge_3_0_0
+from pandas.plotting._matplotlib.converter import register_pandas_matplotlib_converters
 from pandas.plotting._matplotlib.style import _get_standard_colors
 from pandas.plotting._matplotlib.tools import (
     _flatten,
@@ -41,9 +39,6 @@
     table,
 )
 
-if get_option("plotting.matplotlib.register_converters"):
-    converter.register(explicit=False)
-
 
 class MPLPlot:
     """
@@ -112,7 +107,6 @@ def __init__(
 
         import matplotlib.pyplot as plt
 
-        converter._WARN = False  # no warning for pandas plots
         self.data = data
         self.by = by
 
@@ -648,6 +642,7 @@ def _get_xticks(self, convert_period=False):
         return x
 
     @classmethod
+    @register_pandas_matplotlib_converters
     def _plot(cls, ax, x, y, style=None, is_errorbar=False, **kwds):
         mask = isna(y)
         if mask.any():
diff --git a/pandas/plotting/_matplotlib/hist.py b/pandas/plotting/_matplotlib/hist.py
index f95ff2578d882..c4ac9ead3f3d3 100644
--- a/pandas/plotting/_matplotlib/hist.py
+++ b/pandas/plotting/_matplotlib/hist.py
@@ -9,7 +9,6 @@
 import pandas.core.common as com
 
 from pandas.io.formats.printing import pprint_thing
-from pandas.plotting._matplotlib import converter
 from pandas.plotting._matplotlib.core import LinePlot, MPLPlot
 from pandas.plotting._matplotlib.tools import _flatten, _set_ticks_props, _subplots
 
@@ -255,7 +254,6 @@ def _grouped_hist(
     def plot_group(group, ax):
         ax.hist(group.dropna().values, bins=bins, **kwargs)
 
-    converter._WARN = False  # no warning for pandas plots
     xrot = xrot or rot
 
     fig, axes = _grouped_plot(
@@ -363,7 +361,6 @@ def hist_frame(
     bins=10,
     **kwds
 ):
-    converter._WARN = False  # no warning for pandas plots
     if by is not None:
         axes = _grouped_hist(
             data,
diff --git a/pandas/plotting/_matplotlib/timeseries.py b/pandas/plotting/_matplotlib/timeseries.py
index f160e50d8d99b..931c699d9b9fd 100644
--- a/pandas/plotting/_matplotlib/timeseries.py
+++ b/pandas/plotting/_matplotlib/timeseries.py
@@ -25,6 +25,7 @@
     TimeSeries_DateFormatter,
     TimeSeries_DateLocator,
     TimeSeries_TimedeltaFormatter,
+    register_pandas_matplotlib_converters,
 )
 import pandas.tseries.frequencies as frequencies
 from pandas.tseries.offsets import DateOffset
@@ -33,6 +34,7 @@
 # Plotting functions and monkey patches
 
 
+@register_pandas_matplotlib_converters
 def tsplot(series, plotf, ax=None, **kwargs):
     """
     Plots a Series on the given Matplotlib axes or the current axes
@@ -56,7 +58,7 @@ def tsplot(series, plotf, ax=None, **kwargs):
         "'tsplot' is deprecated and will be removed in a "
         "future version. Please use Series.plot() instead.",
         FutureWarning,
-        stacklevel=2,
+        stacklevel=3,
     )
 
     # Used inferred freq is possible, need a test case for inferred
diff --git a/pandas/plotting/_misc.py b/pandas/plotting/_misc.py
index 426ca9632af29..762ee4cf83224 100644
--- a/pandas/plotting/_misc.py
+++ b/pandas/plotting/_misc.py
@@ -30,7 +30,7 @@ def table(ax, data, rowLabels=None, colLabels=None, **kwargs):
     )
 
 
-def register(explicit=True):
+def register():
     """
     Register Pandas Formatters and Converters with matplotlib.
 
@@ -49,7 +49,7 @@ def register(explicit=True):
     deregister_matplotlib_converters
     """
     plot_backend = _get_plot_backend("matplotlib")
-    plot_backend.register(explicit=explicit)
+    plot_backend.register()
 
 
 def deregister():
diff --git a/pandas/tests/plotting/test_converter.py b/pandas/tests/plotting/test_converter.py
index aabe16d5050f9..ccc2afbb8b824 100644
--- a/pandas/tests/plotting/test_converter.py
+++ b/pandas/tests/plotting/test_converter.py
@@ -28,18 +28,6 @@
 pytest.importorskip("matplotlib.pyplot")
 
 
-def test_initial_warning():
-    code = (
-        "import pandas as pd; import matplotlib.pyplot as plt; "
-        "s = pd.Series(1, pd.date_range('2000', periods=12)); "
-        "fig, ax = plt.subplots(); "
-        "ax.plot(s.index, s.values)"
-    )
-    call = [sys.executable, "-c", code]
-    out = subprocess.check_output(call, stderr=subprocess.STDOUT).decode()
-    assert "Using an implicitly" in out
-
-
 def test_registry_mpl_resets():
     # Check that Matplotlib converters are properly reset (see issue #27481)
     code = (
@@ -71,27 +59,12 @@ def test_register_by_default(self):
         call = [sys.executable, "-c", code]
         assert subprocess.check_call(call) == 0
 
-    def test_warns(self):
-        plt = pytest.importorskip("matplotlib.pyplot")
-        s = Series(range(12), index=date_range("2017", periods=12))
-        _, ax = plt.subplots()
-
-        # Set to the "warning" state, in case this isn't the first test run
-        converter._WARN = True
-        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False) as w:
-            ax.plot(s.index, s.values)
-            plt.close()
-
-        assert len(w) == 1
-        assert "Using an implicitly registered datetime converter" in str(w[0])
-
     def test_registering_no_warning(self):
         plt = pytest.importorskip("matplotlib.pyplot")
         s = Series(range(12), index=date_range("2017", periods=12))
         _, ax = plt.subplots()
 
         # Set to the "warn" state, in case this isn't the first test run
-        converter._WARN = True
         register_matplotlib_converters()
         with tm.assert_produces_warning(None) as w:
             ax.plot(s.index, s.values)
@@ -102,7 +75,6 @@ def test_pandas_plots_register(self):
         pytest.importorskip("matplotlib.pyplot")
         s = Series(range(12), index=date_range("2017", periods=12))
         # Set to the "warn" state, in case this isn't the first test run
-        converter._WARN = True
         with tm.assert_produces_warning(None) as w:
             s.plot()
 
@@ -110,13 +82,15 @@ def test_pandas_plots_register(self):
 
     def test_matplotlib_formatters(self):
         units = pytest.importorskip("matplotlib.units")
-        assert Timestamp in units.registry
 
-        ctx = cf.option_context("plotting.matplotlib.register_converters", False)
-        with ctx:
-            assert Timestamp not in units.registry
+        # Can't make any assertion about the start state.
+        # We we check that toggling converters off remvoes it, and toggling it
+        # on restores it.
 
-        assert Timestamp in units.registry
+        with cf.option_context("plotting.matplotlib.register_converters", True):
+            with cf.option_context("plotting.matplotlib.register_converters", False):
+                assert Timestamp not in units.registry
+            assert Timestamp in units.registry
 
     def test_option_no_warning(self):
         pytest.importorskip("matplotlib.pyplot")
@@ -125,7 +99,6 @@ def test_option_no_warning(self):
         s = Series(range(12), index=date_range("2017", periods=12))
         _, ax = plt.subplots()
 
-        converter._WARN = True
         # Test without registering first, no warning
         with ctx:
             with tm.assert_produces_warning(None) as w:
@@ -134,7 +107,6 @@ def test_option_no_warning(self):
         assert len(w) == 0
 
         # Now test with registering
-        converter._WARN = True
         register_matplotlib_converters()
         with ctx:
             with tm.assert_produces_warning(None) as w: