diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 530aaee24c7fb..62842e7286539 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -270,10 +270,6 @@ def _outer_indexer(self, left, right):
     # would we like our indexing holder to defer to us
     _defer_to_indexing = False
 
-    # prioritize current class for _shallow_copy_with_infer,
-    # used to infer integers as datetime-likes
-    _infer_as_myclass = False
-
     _engine_type = libindex.ObjectEngine
     # whether we support partial string indexing. Overridden
     # in DatetimeIndex and PeriodIndex
@@ -441,10 +437,6 @@ def __new__(
       _simple_new), but fills caller's metadata otherwise specified. Passed
       kwargs will overwrite corresponding metadata.
 
-    - _shallow_copy_with_infer: It returns new Index inferring its type
-      from passed values. It fills caller's metadata otherwise specified as the
-      same as _shallow_copy.
-
     See each method's docstring.
     """
 
@@ -517,35 +509,6 @@ def _shallow_copy(self, values=None, name: Label = no_default):
         result._cache = cache
         return result
 
-    def _shallow_copy_with_infer(self, values, **kwargs):
-        """
-        Create a new Index inferring the class with passed value, don't copy
-        the data, use the same object attributes with passed in attributes
-        taking precedence.
-
-        *this is an internal non-public method*
-
-        Parameters
-        ----------
-        values : the values to create the new Index, optional
-        kwargs : updates the default attributes for this Index
-        """
-        attributes = self._get_attributes_dict()
-        attributes.update(kwargs)
-        attributes["copy"] = False
-        if not len(values) and "dtype" not in kwargs:
-            # TODO: what if hasattr(values, "dtype")?
-            attributes["dtype"] = self.dtype
-        if self._infer_as_myclass:
-            try:
-                return self._constructor(values, **attributes)
-            except (TypeError, ValueError):
-                pass
-
-        # Remove tz so Index will try non-DatetimeIndex inference
-        attributes.pop("tz", None)
-        return Index(values, **attributes)
-
     def is_(self, other) -> bool:
         """
         More flexible, faster check like ``is`` but that works through views.
@@ -2810,11 +2773,7 @@ def symmetric_difference(self, other, result_name=None, sort=None):
             except TypeError:
                 pass
 
-        attribs = self._get_attributes_dict()
-        attribs["name"] = result_name
-        if "freq" in attribs:
-            attribs["freq"] = None
-        return self._shallow_copy_with_infer(the_diff, **attribs)
+        return Index(the_diff, dtype=self.dtype, name=result_name)
 
     def _assert_can_do_setop(self, other):
         if not is_list_like(other):
@@ -3388,7 +3347,7 @@ def _reindex_non_unique(self, target):
                 new_indexer = np.arange(len(self.take(indexer)))
                 new_indexer[~check] = -1
 
-        new_index = self._shallow_copy_with_infer(new_labels)
+        new_index = Index(new_labels, name=self.name)
         return new_index, indexer, new_indexer
 
     # --------------------------------------------------------------------
@@ -3945,7 +3904,7 @@ def where(self, cond, other=None):
             # it's float) if there are NaN values in our output.
             dtype = None
 
-        return self._shallow_copy_with_infer(values, dtype=dtype)
+        return Index(values, dtype=dtype, name=self.name)
 
     # construction helpers
     @classmethod
@@ -4175,7 +4134,8 @@ def _concat_same_dtype(self, to_concat, name):
 
         to_concat = [x._values if isinstance(x, Index) else x for x in to_concat]
 
-        return self._shallow_copy_with_infer(np.concatenate(to_concat), **attribs)
+        res_values = np.concatenate(to_concat)
+        return Index(res_values, name=name)
 
     def putmask(self, mask, value):
         """
@@ -5217,7 +5177,7 @@ def insert(self, loc: int, item):
         arr = np.asarray(self)
         item = self._coerce_scalar_to_index(item)._values
         idx = np.concatenate((arr[:loc], item, arr[loc:]))
-        return self._shallow_copy_with_infer(idx)
+        return Index(idx, name=self.name)
 
     def drop(self, labels, errors: str_t = "raise"):
         """
diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py
index ba1a9a4e08fa0..0cf6698d316bb 100644
--- a/pandas/core/indexes/category.py
+++ b/pandas/core/indexes/category.py
@@ -690,7 +690,8 @@ def map(self, mapper):
         >>> idx.map({'a': 'first', 'b': 'second'})
         Index(['first', 'second', nan], dtype='object')
         """
-        return self._shallow_copy_with_infer(self._values.map(mapper))
+        mapped = self._values.map(mapper)
+        return Index(mapped, name=self.name)
 
     def delete(self, loc):
         """
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 1ec6cf8fd7b4e..649d4e6dfc384 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -214,7 +214,6 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
     _attributes = ["name", "tz", "freq"]
 
     _is_numeric_dtype = False
-    _infer_as_myclass = True
 
     _data: DatetimeArray
     tz: Optional[tzinfo]
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
index 42e0d228dab09..52439a1ea3946 100644
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -1046,16 +1046,17 @@ def _shallow_copy(
         result._cache.pop("levels", None)  # GH32669
         return result
 
-    def _shallow_copy_with_infer(self, values, **kwargs):
-        # On equal MultiIndexes the difference is empty.
+    def symmetric_difference(self, other, result_name=None, sort=None):
+        # On equal symmetric_difference MultiIndexes the difference is empty.
         # Therefore, an empty MultiIndex is returned GH13490
-        if len(values) == 0:
+        tups = Index.symmetric_difference(self, other, result_name, sort)
+        if len(tups) == 0:
             return MultiIndex(
                 levels=[[] for _ in range(self.nlevels)],
                 codes=[[] for _ in range(self.nlevels)],
-                **kwargs,
+                names=tups.name,
             )
-        return self._shallow_copy(values, **kwargs)
+        return type(self).from_tuples(tups, names=tups.name)
 
     # --------------------------------------------------------------------
 
diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py
index 1f565828ec7a5..ece656679688f 100644
--- a/pandas/core/indexes/period.py
+++ b/pandas/core/indexes/period.py
@@ -148,7 +148,6 @@ class PeriodIndex(DatetimeIndexOpsMixin, Int64Index):
 
     # define my properties & methods for delegation
     _is_numeric_dtype = False
-    _infer_as_myclass = True
 
     _data: PeriodArray
     freq: DateOffset
diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py
index 765b948f13e96..4caf9107808ed 100644
--- a/pandas/core/indexes/timedeltas.py
+++ b/pandas/core/indexes/timedeltas.py
@@ -121,7 +121,6 @@ class TimedeltaIndex(DatetimeTimedeltaMixin, dtl.TimelikeOps):
     _comparables = ["name", "freq"]
     _attributes = ["name", "freq"]
     _is_numeric_dtype = True
-    _infer_as_myclass = True
 
     _data: TimedeltaArray
 
diff --git a/pandas/core/tools/numeric.py b/pandas/core/tools/numeric.py
index f4eb16602f8a0..72831b29af61e 100644
--- a/pandas/core/tools/numeric.py
+++ b/pandas/core/tools/numeric.py
@@ -188,7 +188,7 @@ def to_numeric(arg, errors="raise", downcast=None):
         return pd.Series(values, index=arg.index, name=arg.name)
     elif is_index:
         # because we want to coerce to numeric if possible,
-        # do not use _shallow_copy_with_infer
+        # do not use _shallow_copy
         return pd.Index(values, name=arg.name)
     elif is_scalars:
         return values[0]