BUG: infer-objects raising for bytes Series (#50067)

* BUG: infer-objects raising for bytes Series

* Merge remote-tracking branch 'upstream/main' into 49650

# Conflicts:
#	pandas/tests/series/methods/test_infer_objects.py

Author: phofl

pandas/core/internals/blocks.py

@@ -1969,6 +1969,9 @@ def convert(
         attempt to cast any object types to better types return a copy of
         the block (if copy = True) by definition we ARE an ObjectBlock!!!!!
         """
+        if self.dtype != object:
+            return [self]
+
         values = self.values
         if values.ndim == 2:
             # maybe_split ensures we only get here with values.shape[0] == 1,

pandas/tests/series/methods/test_infer_objects.py

@@ -1,6 +1,9 @@
 import numpy as np
 
-from pandas import interval_range
+from pandas import (
+    Series,
+    interval_range,
+)
 import pandas._testing as tm
 
 
@@ -44,3 +47,10 @@ def test_infer_objects_interval(self, index_or_series):
 
         result = obj.astype(object).infer_objects()
         tm.assert_equal(result, obj)
+
+    def test_infer_objects_bytes(self):
+        # GH#49650
+        ser = Series([b"a"], dtype="bytes")
+        expected = ser.copy()
+        result = ser.infer_objects()
+        tm.assert_series_equal(result, expected)