ENH: Added DataFrame.compare and Series.compare (GH30429)

pandas/core/frame.py

@@ -5365,6 +5365,94 @@ def _construct_result(self, result) -> "DataFrame":
         out.columns = self.columns
         return out
 
+    @Appender(
+        """
+Returns
+-------
+DataFrame
+    DataFrame that shows the differences stacked side by side.
+
+See Also
+--------
+Series.differences: Show differences.
+
+Examples
+--------
+>>> df = pd.DataFrame(
+...     {
+...         "col1": ["a", "a", "b", "b", "a"],
+...         "col2": [1.0, 2.0, 3.0, np.nan, 5.0],
+...         "col3": [1.0, 2.0, 3.0, 4.0, 5.0]
+...     },
+...     columns=["col1", "col2", "col3"],
+... )
+>>> df
+  col1  col2  col3
+0    a   1.0   1.0
+1    a   2.0   2.0
+2    b   3.0   3.0
+3    b   NaN   4.0
+4    a   5.0   5.0
+
+>>> df2 = df.copy()
+>>> df2.loc[0, 'col1'] = 'c'
+>>> df2.loc[2, 'col3'] = 4.0
+>>> df2
+  col1  col2  col3
+0    c   1.0   1.0
+1    a   2.0   2.0
+2    b   3.0   4.0
+3    b   NaN   4.0
+4    a   5.0   5.0
+
+Stack the differences on columns
+
+>>> df.differences(df2)
+  col1       col3
+  self other self other
+0    a     c  NaN   NaN
+2  NaN   NaN  3.0   4.0
+
+Stack the differences on rows
+
+>>> df.differences(df2, axis=0)
+        col1  col3
+0 self     a   NaN
+  other    c   NaN
+2 self   NaN   3.0
+  other  NaN   4.0
+
+Keep all the original indices (rows and columns)
+
+>>> df.differences(df2, keep_indices=True)
+  col1       col2       col3
+  self other self other self other
+0    a     c  NaN   NaN  NaN   NaN
+1  NaN   NaN  NaN   NaN  NaN   NaN
+2  NaN   NaN  NaN   NaN  3.0   4.0
+3  NaN   NaN  NaN   NaN  NaN   NaN
+4  NaN   NaN  NaN   NaN  NaN   NaN
+
+Keep all original indices and data
+
+>>> df.differences(df2, keep_indices=True, keep_values=True)
+  col1       col2       col3
+  self other self other self other
+0    a     c  1.0   1.0  1.0   1.0
+1    a     a  2.0   2.0  2.0   2.0
+2    b     b  3.0   3.0  3.0   4.0
+3    b     b  NaN   NaN  4.0   4.0
+4    a     a  5.0   5.0  5.0   5.0
+"""
+    )
+    @Appender(_shared_docs["differences"] % _shared_doc_kwargs)
+    def differences(
+        self, other: "DataFrame", axis=1, keep_indices=False, keep_values=False
+    ) -> "DataFrame":
+        return super().differences(
+            other=other, axis=axis, keep_indices=keep_indices, keep_values=keep_values
+        )
+
     def combine(
         self, other: "DataFrame", func, fill_value=None, overwrite=True
     ) -> "DataFrame":

pandas/core/generic.py

@@ -8103,6 +8103,90 @@ def ranker(data):
 
         return ranker(data)
 
+    _shared_docs[
+        "differences"
+    ] = """
+        Compare to another %(klass)s and show the differences.
+
+        The axis on which to stack results and how much information to
+        preserve can be customized.
+
+        Note that NaNs are considered not different from other NaNs.
+
+        Parameters
+        ----------
+        other : %(klass)s
+            Object to compare with.
+
+        axis : {0 or 'index', 1 or 'columns'}, default 1
+            Determine how the differences are stacked.
+            * 0, or 'index' : Stack differences on neighbouring rows.
+            * 1, or 'columns' : Stack differences on neighbouring columns.
+
+        keep_indices : bool, default False
+            Whether to keep the rows and columns that are equal, or drop them.
+
+        keep_values : bool, default False
+            Whether to keep the values that are equal, or show as NaNs.
+        """
+
+    @Appender(_shared_docs["differences"] % _shared_doc_kwargs)
+    def differences(self, other, axis=1, keep_indices=False, keep_values=False):
+        from pandas.core.reshape.concat import concat
+
+        mask = ~((self == other) | (self.isna() & other.isna()))
+        keys = ["self", "other"]
+
+        if not keep_values:
+            self = self.where(mask)
+            other = other.where(mask)
+
+        if not keep_indices:
+            if isinstance(self, ABCDataFrame):
+                cmask = mask.any()
+                rmask = mask.any(axis=1)
+                self = self.loc[rmask, cmask]
+                other = other.loc[rmask, cmask]
+            else:
+                self = self[mask]
+                other = other[mask]
+
+        if axis in (1, "columns"):  # This is needed for Series
+            axis = 1
+        else:
+            axis = self._get_axis_number(axis)
+
+        diff = concat([self, other], axis=axis, keys=keys)
+
+        if axis >= self.ndim:
+            # No need to reorganize data if stacking on new axis
+            # This currently applies for stacking two Series on columns
+            return diff
+
+        ax = diff._get_axis(axis)
+        ax_names = np.array(ax.names)
+
+        # set index names to positions to avoid confusion
+        ax.names = np.arange(len(ax_names))
+
+        # bring self-other to inner level
+        order = list(range(1, ax.nlevels)) + [0]
+        if isinstance(diff, ABCDataFrame):
+            diff = diff.reorder_levels(order, axis=axis)
+        else:
+            diff = diff.reorder_levels(order)
+
+        # restore the index names in order
+        diff._get_axis(axis=axis).names = ax_names[order]
+
+        # reorder axis to keep things organized
+        indices = (
+            np.arange(diff.shape[axis]).reshape([2, diff.shape[axis] // 2]).T.flatten()
+        )
+        diff = diff.take(indices, axis=axis)
+
+        return diff
+
     _shared_docs[
         "align"
     ] = """

pandas/core/series.py

@@ -23,7 +23,7 @@
 from pandas._config import get_option
 
 from pandas._libs import index as libindex, lib, reshape, tslibs
-from pandas._typing import Label
+from pandas._typing import FrameOrSeries, Label
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import Appender, Substitution
 from pandas.util._validators import validate_bool_kwarg, validate_percentile
@@ -2555,7 +2555,70 @@ def _binop(self, other, func, level=None, fill_value=None):
         ret = ops._construct_result(self, result, new_index, name)
         return ret
 
+    @Appender(
+        """
+Returns
+-------
+Series or DataFrame
+    If axis is 0 or 'index' the result will be a Series.
+    If axis is 1 or 'columns' the result will be a DataFrame.
+
+See Also
+--------
+DataFrame.differences: Show differences.
+
+Examples
+--------
+>>> s1 = pd.Series(["a", "b", "c", "d", "e"])
+>>> s2 = pd.Series(["a", "a", "c", "b", "e"])
+
+Stack the differences on columns
+
+>>> s1.differences(s2)
+  self other
+1    b     a
+3    d     b
+
+Stack the differences on indices
+
+>>> s1.differences(s2, axis=0)
+1  self     b
+   other    a
+3  self     d
+   other    b
+dtype: object
+
+Keep all the original indices
+
+>>> s1.differences(s2, keep_indices=True)
+  self other
+0  NaN   NaN
+1    b     a
+2  NaN   NaN
+3    d     b
+4  NaN   NaN
+
+Keep all original indices and data
+
+>>> s1.differences(s2, keep_indices=True, keep_values=True)
+  self other
+0    a     a
+1    b     a
+2    c     c
+3    d     b
+4    e     e
+"""
+    )
+    @Appender(generic._shared_docs["differences"] % _shared_doc_kwargs)
+    def differences(
+        self, other: "Series", axis=1, keep_indices=False, keep_values=False
+    ) -> FrameOrSeries:
+        return super().differences(
+            other=other, axis=axis, keep_indices=keep_indices, keep_values=keep_values
+        )
+
     def combine(self, other, func, fill_value=None) -> "Series":
+
         """
         Combine the Series with a Series or scalar according to `func`.
 

pandas/tests/frame/methods/test_differences.py

@@ -0,0 +1,130 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+import pandas._testing as tm
+
+
+@pytest.mark.parametrize("axis", [0, 1, "index", "columns"])
+def test_differences_axis(axis):
+    df = pd.DataFrame(
+        {"col1": ["a", "b", "c"], "col2": [1.0, 2.0, np.nan], "col3": [1.0, 2.0, 3.0]},
+        columns=["col1", "col2", "col3"],
+    )
+    df2 = df.copy()
+    df2.loc[0, "col1"] = "c"
+    df2.loc[2, "col3"] = 4.0
+
+    result = df.differences(df2, axis=axis)
+
+    if axis in (1, "columns"):
+        indices = pd.Index([0, 2])
+        columns = pd.MultiIndex.from_product([["col1", "col3"], ["self", "other"]])
+        expected = pd.DataFrame(
+            [["a", "c", np.nan, np.nan], [np.nan, np.nan, 3.0, 4.0]],
+            index=indices,
+            columns=columns,
+        )
+    else:
+        indices = pd.MultiIndex.from_product([[0, 2], ["self", "other"]])
+        columns = pd.Index(["col1", "col3"])
+        expected = pd.DataFrame(
+            [["a", np.nan], ["c", np.nan], [np.nan, 3.0], [np.nan, 4.0]],
+            index=indices,
+            columns=columns,
+        )
+    tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+    "keep_indices, keep_values",
+    [
+        (True, False),
+        (False, True),
+        (True, True),
+        # False, False case is already covered in test_differences_axis
+    ],
+)
+def test_differences_various_formats(keep_indices, keep_values):
+    df = pd.DataFrame(
+        {"col1": ["a", "b", "c"], "col2": [1.0, 2.0, np.nan], "col3": [1.0, 2.0, 3.0]},
+        columns=["col1", "col2", "col3"],
+    )
+    df2 = df.copy()
+    df2.loc[0, "col1"] = "c"
+    df2.loc[2, "col3"] = 4.0
+
+    result = df.differences(df2, keep_indices=keep_indices, keep_values=keep_values)
+
+    if keep_indices:
+        indices = pd.Index([0, 1, 2])
+        columns = pd.MultiIndex.from_product(
+            [["col1", "col2", "col3"], ["self", "other"]]
+        )
+        if keep_values:
+            expected = pd.DataFrame(
+                [
+                    ["a", "c", 1.0, 1.0, 1.0, 1.0],
+                    ["b", "b", 2.0, 2.0, 2.0, 2.0],
+                    ["c", "c", np.nan, np.nan, 3.0, 4.0],
+                ],
+                index=indices,
+                columns=columns,
+            )
+        else:
+            expected = pd.DataFrame(
+                [
+                    ["a", "c", np.nan, np.nan, np.nan, np.nan],
+                    [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
+                    [np.nan, np.nan, np.nan, np.nan, 3.0, 4.0],
+                ],
+                index=indices,
+                columns=columns,
+            )
+    else:
+        indices = pd.Index([0, 2])
+        columns = pd.MultiIndex.from_product([["col1", "col3"], ["self", "other"]])
+        expected = pd.DataFrame(
+            [["a", "c", 1.0, 1.0], ["c", "c", 3.0, 4.0]], index=indices, columns=columns
+        )
+    tm.assert_frame_equal(result, expected)
+
+
+def test_differences_with_equal_nulls():
+    # We want to make sure two NaNs are considered the same
+    # and dropped where applicable
+    df = pd.DataFrame(
+        {"col1": ["a", "b", "c"], "col2": [1.0, 2.0, np.nan], "col3": [1.0, 2.0, 3.0]},
+        columns=["col1", "col2", "col3"],
+    )
+    df2 = df.copy()
+    df2.loc[0, "col1"] = "c"
+
+    result = df.differences(df2)
+    indices = pd.Index([0])
+    columns = pd.MultiIndex.from_product([["col1"], ["self", "other"]])
+    expected = pd.DataFrame([["a", "c"]], index=indices, columns=columns)
+    tm.assert_frame_equal(result, expected)
+
+
+def test_differences_with_non_equal_nulls():
+    # We want to make sure the relevant NaNs do not get dropped
+    # even if the entire row or column are NaNs
+    df = pd.DataFrame(
+        {"col1": ["a", "b", "c"], "col2": [1.0, 2.0, np.nan], "col3": [1.0, 2.0, 3.0]},
+        columns=["col1", "col2", "col3"],
+    )
+    df2 = df.copy()
+    df2.loc[0, "col1"] = "c"
+    df2.loc[2, "col3"] = np.nan
+
+    result = df.differences(df2)
+
+    indices = pd.Index([0, 2])
+    columns = pd.MultiIndex.from_product([["col1", "col3"], ["self", "other"]])
+    expected = pd.DataFrame(
+        [["a", "c", np.nan, np.nan], [np.nan, np.nan, 3.0, np.nan]],
+        index=indices,
+        columns=columns,
+    )
+    tm.assert_frame_equal(result, expected)