diff --git a/hoverxref/extension.py b/hoverxref/extension.py
index c227ecf0..74d80edb 100644
--- a/hoverxref/extension.py
+++ b/hoverxref/extension.py
@@ -173,8 +173,7 @@ def setup_intersphinx(app, config):
         return
 
     for listener in app.events.listeners.get('missing-reference'):
-        module_name = inspect.getmodule(listener.handler).__name__
-        if module_name == 'sphinx.ext.intersphinx':
+        if listener.handler == sphinx_missing_reference:
             app.disconnect(listener.id)
 
 
diff --git a/tests/test_internals.py b/tests/test_internals.py
index 509766d7..2d4e57ce 100644
--- a/tests/test_internals.py
+++ b/tests/test_internals.py
@@ -1,9 +1,10 @@
-import inspect
-import os
 import pytest
-import shutil
 from unittest import mock
 
+from sphinx.events import EventListener
+from sphinx.ext.intersphinx import missing_reference as intersphinx_missing_reference
+from hoverxref.extension import missing_reference
+
 from .utils import srcdir
 
 
@@ -34,3 +35,23 @@ def _get_docpath(self, builder, docname):
     content = open(path).read()
 
     assert app.builder.format == 'latex'
+
+
+@pytest.mark.sphinx(
+    srcdir=srcdir,
+    confoverrides={
+        'hoverxref_domains': ['py'],
+        'hoverxref_intersphinx': ['python'],
+        'hoverxref_auto_ref': True,
+        'extensions': [
+            'sphinx.ext.intersphinx',
+            'hoverxref.extension',
+        ],
+    },
+)
+def test_disconnect_intersphinx_listener(app, status, warning):
+    """The ``missing-reference`` listener from ``sphinx.ext.intershinx`` should be dropped in favor of ours."""
+    app.build()
+    missing_reference_listeners = app.events.listeners['missing-reference']
+    assert EventListener(id=mock.ANY, priority=mock.ANY, handler=intersphinx_missing_reference) not in missing_reference_listeners
+    assert EventListener(id=mock.ANY, priority=mock.ANY, handler=missing_reference) in missing_reference_listeners
diff --git a/tox.ini b/tox.ini
index 938d57d1..6e37deba 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@ isolated_build = True
 
 envlist =
   docs
-  py{38,39,310,312}-sphinx{50,53,60,62,70,73,latest}
+  py{38,39,310,312}-sphinx{50,53,60,62,70,73,74,latest}
 
 [testenv]
 deps =
@@ -18,6 +18,7 @@ deps =
   sphinx62: sphinx~=6.2.0
   sphinx70: sphinx~=7.0.0
   sphinx73: sphinx[test]~=7.3.0
+  sphinx74: sphinx[test]~=7.4.0
   sphinxlatest: sphinx[test]
 
 commands = pytest {posargs}