move inferred_type check

Author: topper-123

pandas/_testing/asserters.py

@@ -314,16 +314,15 @@ def _check_types(left, right, obj="Index") -> None:
             return
 
         assert_class_equal(left, right, exact=exact, obj=obj)
+        assert_attr_equal("inferred_type", left, right, obj=obj)
 
         # Skip exact dtype checking when `check_categorical` is False
         if is_categorical_dtype(left.dtype) and is_categorical_dtype(right.dtype):
             if check_categorical:
                 assert_attr_equal("dtype", left, right, obj=obj)
                 assert_index_equal(left.categories, right.categories, exact=exact)
-            assert_attr_equal("inferred_type", left, right, obj=obj)
             return
 
-        assert_attr_equal("inferred_type", left, right, obj=obj)
         assert_attr_equal("dtype", left, right, obj=obj)
 
     def _get_ilevel_values(index, level):