diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst
index 6b8d548251061..11673f332876f 100644
--- a/doc/source/whatsnew/v0.24.0.rst
+++ b/doc/source/whatsnew/v0.24.0.rst
@@ -1209,6 +1209,7 @@ Performance Improvements
 - Improved performance of :func:`Series.describe` in case of numeric dtpyes (:issue:`21274`)
 - Improved performance of :func:`pandas.core.groupby.GroupBy.rank` when dealing with tied rankings (:issue:`21237`)
 - Improved performance of :func:`DataFrame.set_index` with columns consisting of :class:`Period` objects (:issue:`21582`, :issue:`21606`)
+- Improved performance of :meth:`Series.at` and :meth:`Index.get_value` for Extension Arrays values (e.g. :class:`Categorical`) (:issue:`24204`)
 - Improved performance of membership checks in :class:`Categorical` and :class:`CategoricalIndex`
   (i.e. ``x in cat``-style checks are much faster). :meth:`CategoricalIndex.contains`
   is likewise much faster (:issue:`21369`, :issue:`21508`)
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index fc5f6758f9e06..811d66c74ed15 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -4281,7 +4281,8 @@ def get_value(self, series, key):
 
         # if we have something that is Index-like, then
         # use this, e.g. DatetimeIndex
-        s = getattr(series, '_values', None)
+        # Things like `Series._get_value` (via .at) pass the EA directly here.
+        s = getattr(series, '_values', series)
         if isinstance(s, (ExtensionArray, Index)) and is_scalar(key):
             # GH 20882, 21257
             # Unify Index and ExtensionArray treatment
diff --git a/pandas/tests/arrays/categorical/test_indexing.py b/pandas/tests/arrays/categorical/test_indexing.py
index 8df5728f7d895..8b31bb0534368 100644
--- a/pandas/tests/arrays/categorical/test_indexing.py
+++ b/pandas/tests/arrays/categorical/test_indexing.py
@@ -145,3 +145,30 @@ def test_mask_with_boolean_raises(index):
 
     with pytest.raises(ValueError, match='NA / NaN'):
         s[idx]
+
+
+@pytest.fixture
+def non_coercible_categorical(monkeypatch):
+    """
+    Monkeypatch Categorical.__array__ to ensure no implicit conversion.
+
+    Raises
+    ------
+    ValueError
+        When Categorical.__array__ is called.
+    """
+    # TODO(Categorical): identify other places where this may be
+    # useful and move to a conftest.py
+    def array(self, dtype=None):
+        raise ValueError("I cannot be converted.")
+
+    with monkeypatch.context() as m:
+        m.setattr(Categorical, "__array__", array)
+        yield
+
+
+def test_series_at(non_coercible_categorical):
+    arr = Categorical(['a', 'b', 'c'])
+    ser = Series(arr)
+    result = ser.at[0]
+    assert result == 'a'