BUG: Allow pairwise calcuation when comparing the column with itself (pandas-dev#25781)

Author: liwh

pandas/core/frame.py

@@ -9396,6 +9396,7 @@ def corr(
         self,
         method: str | Callable[[np.ndarray, np.ndarray], float] = "pearson",
         min_periods: int = 1,
+        calculate_diagonal: bool = False,
     ) -> DataFrame:
         """
         Compute pairwise correlation of columns, excluding NA/null values.
@@ -9416,6 +9417,10 @@ def corr(
             Minimum number of observations required per pair of columns
             to have a valid result. Currently only available for Pearson
             and Spearman correlation.
+        calculate_diagonal : bool, optional
+            Whether to calculate pairwise correlation using supplied callable.
+            Ignored when method argument is not callable. If False, pairwise
+            correlation between a column and itself is default to 1.
 
         Returns
         -------
@@ -9465,7 +9470,7 @@ def corr(
                     valid = mask[i] & mask[j]
                     if valid.sum() < min_periods:
                         c = np.nan
-                    elif i == j:
+                    elif i == j and not calculate_diagonal:
                         c = 1.0
                     elif not valid.all():
                         c = corrf(ac[valid], bc[valid])

pandas/tests/frame/methods/test_cov_corr.py

@@ -236,6 +236,21 @@ def test_corr_min_periods_greater_than_length(self, method):
         )
         tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.filterwarnings("ignore: An input array is constant")
+    @td.skip_if_no_scipy
+    @pytest.mark.parametrize("array_creator", [np.ones, np.zeros, np.random.random])
+    def test_corr_diagonal_not_ones(self, array_creator):
+        from scipy.stats import pearsonr
+
+        frame_size = 4
+        df = DataFrame(array_creator((frame_size, frame_size)))
+        cor_mat = df.corr(
+            method=lambda x, y: pearsonr(x, y)[0], calculate_diagonal=True
+        )
+        result_diag = [cor_mat.loc[i, i] for i in range(frame_size)]
+        expected_diag = [pearsonr(df[i], df[i])[0] for i in range(frame_size)]
+        tm.assert_almost_equal(result_diag, expected_diag)
+
 
 class TestDataFrameCorrWith:
     def test_corrwith(self, datetime_frame):