diff --git a/doc/source/whatsnew/v0.25.1.rst b/doc/source/whatsnew/v0.25.1.rst
index 4e1bfac77fae2..b39dd30003d50 100644
--- a/doc/source/whatsnew/v0.25.1.rst
+++ b/doc/source/whatsnew/v0.25.1.rst
@@ -110,6 +110,9 @@ Plotting
 ^^^^^^^^
 
 - Added a pandas_plotting_backends entrypoint group for registering plot backends. See :ref:`extending.plotting-backends` for more (:issue:`26747`).
+- Fixed the re-instatement of Matplotlib datetime converters after calling
+  `pandas.plotting.deregister_matplotlib_converters()` (:issue:`27481`).
+-
 - Fix compatibility issue with matplotlib when passing a pandas ``Index`` to a plot call (:issue:`27775`).
 -
 
diff --git a/pandas/plotting/_matplotlib/converter.py b/pandas/plotting/_matplotlib/converter.py
index 15648d59c8f98..893854ab26e37 100644
--- a/pandas/plotting/_matplotlib/converter.py
+++ b/pandas/plotting/_matplotlib/converter.py
@@ -64,11 +64,12 @@ def register(explicit=True):
 
     pairs = get_pairs()
     for type_, cls in pairs:
-        converter = cls()
-        if type_ in units.registry:
+        # Cache previous converter if present
+        if type_ in units.registry and not isinstance(units.registry[type_], cls):
             previous = units.registry[type_]
             _mpl_units[type_] = previous
-        units.registry[type_] = converter
+        # Replace with pandas converter
+        units.registry[type_] = cls()
 
 
 def deregister():
diff --git a/pandas/tests/plotting/test_converter.py b/pandas/tests/plotting/test_converter.py
index 35d12706f0590..7001264c41c05 100644
--- a/pandas/tests/plotting/test_converter.py
+++ b/pandas/tests/plotting/test_converter.py
@@ -40,6 +40,21 @@ def test_initial_warning():
     assert "Using an implicitly" in out
 
 
+def test_registry_mpl_resets():
+    # Check that Matplotlib converters are properly reset (see issue #27481)
+    code = (
+        "import matplotlib.units as units; "
+        "import matplotlib.dates as mdates; "
+        "n_conv = len(units.registry); "
+        "import pandas as pd; "
+        "pd.plotting.register_matplotlib_converters(); "
+        "pd.plotting.deregister_matplotlib_converters(); "
+        "assert len(units.registry) == n_conv"
+    )
+    call = [sys.executable, "-c", code]
+    subprocess.check_output(call)
+
+
 def test_timtetonum_accepts_unicode():
     assert converter.time2num("00:01") == converter.time2num("00:01")