diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst
index db23bfdc8a5bd..ef7880617c869 100644
--- a/doc/source/whatsnew/v1.0.0.rst
+++ b/doc/source/whatsnew/v1.0.0.rst
@@ -564,6 +564,7 @@ Indexing
 - Fix assignment of column via `.loc` with numpy non-ns datetime type (:issue:`27395`)
 - Bug in :meth:`Float64Index.astype` where ``np.inf`` was not handled properly when casting to an integer dtype (:issue:`28475`)
 - :meth:`Index.union` could fail when the left contained duplicates (:issue:`28257`)
+- Bug when indexing with ``.loc`` and the index is a :class:`CateggoricalIndex` with integer categories (:issue:`17569`)
 - :meth:`Index.get_indexer_non_unique` could fail with `TypeError` in some cases, such as when searching for ints in a string index (:issue:`28257`)
 - Bug in :meth:`Float64Index.get_loc` incorrectly raising ``TypeError`` instead of ``KeyError`` (:issue:`29189`)
 
diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py
index d061f61effff3..7552912e04ed5 100644
--- a/pandas/core/indexes/category.py
+++ b/pandas/core/indexes/category.py
@@ -696,7 +696,7 @@ def get_indexer_non_unique(self, target):
 
     @Appender(_index_shared_docs["_convert_scalar_indexer"])
     def _convert_scalar_indexer(self, key, kind=None):
-        if self.categories._defer_to_indexing:
+        if kind == "loc" or self.categories._defer_to_indexing:
             return self.categories._convert_scalar_indexer(key, kind=kind)
 
         return super()._convert_scalar_indexer(key, kind=kind)
diff --git a/pandas/tests/indexing/test_categorical.py b/pandas/tests/indexing/test_categorical.py
index ab3b0ed13b5c0..9d0421b2e9ec9 100644
--- a/pandas/tests/indexing/test_categorical.py
+++ b/pandas/tests/indexing/test_categorical.py
@@ -754,3 +754,26 @@ def test_map_with_dict_or_series(self):
         output = cur_index.map(mapper)
         # Order of categories in output can be different
         tm.assert_index_equal(expected, output)
+
+    def test_indexing_with_integer_categories(self):
+        # GH-17569
+        cat_idx = CategoricalIndex([1, 2, 3])
+        cat = DataFrame({"A": ["foo", "bar", "baz"]}, index=cat_idx)
+        # scalar
+        result = cat.loc[1]
+        expected = Series(["foo"], index=["A"], name=1)
+        tm.assert_series_equal(result, expected)
+        # list
+        result = cat.loc[[1, 2]]
+        expected = DataFrame(["foo", "bar"], index=cat_idx[:2], columns=["A"])
+        tm.assert_frame_equal(result, expected)
+        # scalar assignment
+        result = cat.copy()
+        result.loc[1] = "qux"
+        expected = DataFrame({"A": ["qux", "bar", "baz"]}, index=cat_idx)
+        tm.assert_frame_equal(result, expected)
+        # list assignment
+        result = cat.copy()
+        result.loc[[1, 2], "A"] = ["qux", "qux2"]
+        expected = DataFrame({"A": ["qux", "qux2", "baz"]}, index=cat_idx)
+        tm.assert_frame_equal(result, expected)