diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py
index 03bc76791f1..e6dd187540b 100644
--- a/readthedocs/vcs_support/backends/git.py
+++ b/readthedocs/vcs_support/backends/git.py
@@ -203,16 +203,28 @@ def clone(self):
     def tags(self):
         versions = []
         repo = git.Repo(self.working_dir)
+
+        # Build a cache of tag -> commit
+        # GitPython is not very optimized for reading large numbers of tags
+        ref_cache = {}  # 'ref/tags/<tag>' -> hexsha
+        for hexsha, ref in git.TagReference._iter_packed_refs(repo):
+            ref_cache[ref] = hexsha
+
         for tag in repo.tags:
-            try:
-                versions.append(VCSVersion(self, str(tag.commit), str(tag)))
-            except ValueError:
-                # ValueError: Cannot resolve commit as tag TAGNAME points to a
-                # blob object - use the `.object` property instead to access it
-                # This is not a real tag for us, so we skip it
-                # https://github.com/rtfd/readthedocs.org/issues/4440
-                log.warning('Git tag skipped: %s', tag, exc_info=True)
-                continue
+            if tag.path in ref_cache:
+                hexsha = ref_cache[tag.path]
+            else:
+                try:
+                    hexsha = str(tag.commit)
+                except ValueError:
+                    # ValueError: Cannot resolve commit as tag TAGNAME points to a
+                    # blob object - use the `.object` property instead to access it
+                    # This is not a real tag for us, so we skip it
+                    # https://github.com/rtfd/readthedocs.org/issues/4440
+                    log.warning('Git tag skipped: %s', tag, exc_info=True)
+                    continue
+
+            versions.append(VCSVersion(self, hexsha, str(tag)))
         return versions
 
     @property