BUG: frame[x].loc[y] inconsistent with frame.at[x, y] GH#22372 (#45287)

Co-authored-by: Jeff Reback <[email protected]>

Author: jbrockmendel

doc/source/whatsnew/v1.5.0.rst

@@ -200,6 +200,7 @@ Interval
 
 Indexing
 ^^^^^^^^
+- Bug in :meth:`loc.__getitem__` with a list of keys causing an internal inconsistency that could lead to a disconnect between ``frame.at[x, y]`` vs ``frame[y].loc[x]`` (:issue:`22372`)
 - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`)
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
 - Bug when setting an integer too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`)

pandas/core/internals/managers.py

@@ -671,9 +671,6 @@ def reindex_indexer(
             result.axes[axis] = new_axis
             return result
 
-        if consolidate:
-            self._consolidate_inplace()
-
         # some axes don't allow reindexing with dups
         if not allow_dups:
             self.axes[axis]._validate_can_reindex(indexer)
@@ -1681,6 +1678,10 @@ def _consolidate_check(self) -> None:
         self._known_consolidated = True
 
     def _consolidate_inplace(self) -> None:
+        # In general, _consolidate_inplace should only be called via
+        #  DataFrame._consolidate_inplace, otherwise we will fail to invalidate
+        #  the DataFrame's _item_cache. The exception is for newly-created
+        #  BlockManager objects not yet attached to a DataFrame.
         if not self.is_consolidated():
             self.blocks = tuple(_consolidate(self.blocks))
             self._is_consolidated = True

pandas/tests/indexing/test_at.py

@@ -48,6 +48,29 @@ def test_selection_methods_of_assigned_col():
 
 
 class TestAtSetItem:
+    def test_at_setitem_item_cache_cleared(self):
+        # GH#22372 Note the multi-step construction is necessary to trigger
+        #  the original bug. pandas/issues/22372#issuecomment-413345309
+        df = DataFrame(index=[0])
+        df["x"] = 1
+        df["cost"] = 2
+
+        # accessing df["cost"] adds "cost" to the _item_cache
+        df["cost"]
+
+        # This loc[[0]] lookup used to call _consolidate_inplace at the
+        #  BlockManager level, which failed to clear the _item_cache
+        df.loc[[0]]
+
+        df.at[0, "x"] = 4
+        df.at[0, "cost"] = 789
+
+        expected = DataFrame({"x": [4], "cost": 789}, index=[0])
+        tm.assert_frame_equal(df, expected)
+
+        # And in particular, check that the _item_cache has updated correctly.
+        tm.assert_series_equal(df["cost"], expected["cost"])
+
     def test_at_setitem_mixed_index_assignment(self):
         # GH#19860
         ser = Series([1, 2, 3, 4, 5], index=["a", "b", "c", 1, 2])

pandas/tests/internals/test_internals.py

@@ -722,7 +722,10 @@ def test_reindex_items(self):
         mgr = create_mgr("a: f8; b: i8; c: f8; d: i8; e: f8; f: bool; g: f8-2")
 
         reindexed = mgr.reindex_axis(["g", "c", "a", "d"], axis=0)
-        assert reindexed.nblocks == 2
+        # reindex_axis does not consolidate_inplace, as that risks failing to
+        #  invalidate _item_cache
+        assert not reindexed.is_consolidated()
+
         tm.assert_index_equal(reindexed.items, Index(["g", "c", "a", "d"]))
         tm.assert_almost_equal(
             mgr.iget(6).internal_values(), reindexed.iget(0).internal_values()