Backport PR #45287 on branch 1.4.x (BUG: frame[x].loc[y] inconsistent with frame.at[x, y]) (#45287) (#47921)

* BUG: frame[x].loc[y] inconsistent with frame.at[x, y] GH#22372 (#45287)

Co-authored-by: Jeff Reback <[email protected]>

* Add whatsnew

* Remove whatsnew file

* Move whatsnew

Co-authored-by: jbrockmendel <[email protected]>
Co-authored-by: Jeff Reback <[email protected]>

Author: 3 people

doc/source/whatsnew/v1.4.4.rst

@@ -26,6 +26,7 @@ Bug fixes
 ~~~~~~~~~
 - The :class:`errors.FutureWarning` raised when passing arguments (other than ``filepath_or_buffer``) as positional in :func:`read_csv` is now raised at the correct stacklevel (:issue:`47385`)
 - Bug in :meth:`DataFrame.to_sql` when ``method`` was a ``callable`` that did not return an ``int`` and would raise a ``TypeError`` (:issue:`46891`)
+- Bug in :meth:`loc.__getitem__` with a list of keys causing an internal inconsistency that could lead to a disconnect between ``frame.at[x, y]`` vs ``frame[y].loc[x]`` (:issue:`22372`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/internals/managers.py

@@ -671,9 +671,6 @@ def reindex_indexer(
             result.axes[axis] = new_axis
             return result
 
-        if consolidate:
-            self._consolidate_inplace()
-
         # some axes don't allow reindexing with dups
         if not allow_dups:
             self.axes[axis]._validate_can_reindex(indexer)
@@ -1681,6 +1678,10 @@ def _consolidate_check(self) -> None:
         self._known_consolidated = True
 
     def _consolidate_inplace(self) -> None:
+        # In general, _consolidate_inplace should only be called via
+        #  DataFrame._consolidate_inplace, otherwise we will fail to invalidate
+        #  the DataFrame's _item_cache. The exception is for newly-created
+        #  BlockManager objects not yet attached to a DataFrame.
         if not self.is_consolidated():
             self.blocks = tuple(_consolidate(self.blocks))
             self._is_consolidated = True

pandas/tests/indexing/test_at.py

@@ -48,6 +48,29 @@ def test_selection_methods_of_assigned_col():
 
 
 class TestAtSetItem:
+    def test_at_setitem_item_cache_cleared(self):
+        # GH#22372 Note the multi-step construction is necessary to trigger
+        #  the original bug. pandas/issues/22372#issuecomment-413345309
+        df = DataFrame(index=[0])
+        df["x"] = 1
+        df["cost"] = 2
+
+        # accessing df["cost"] adds "cost" to the _item_cache
+        df["cost"]
+
+        # This loc[[0]] lookup used to call _consolidate_inplace at the
+        #  BlockManager level, which failed to clear the _item_cache
+        df.loc[[0]]
+
+        df.at[0, "x"] = 4
+        df.at[0, "cost"] = 789
+
+        expected = DataFrame({"x": [4], "cost": 789}, index=[0])
+        tm.assert_frame_equal(df, expected)
+
+        # And in particular, check that the _item_cache has updated correctly.
+        tm.assert_series_equal(df["cost"], expected["cost"])
+
     def test_at_setitem_mixed_index_assignment(self):
         # GH#19860
         ser = Series([1, 2, 3, 4, 5], index=["a", "b", "c", 1, 2])

pandas/tests/internals/test_internals.py

@@ -718,7 +718,10 @@ def test_reindex_items(self):
         mgr = create_mgr("a: f8; b: i8; c: f8; d: i8; e: f8; f: bool; g: f8-2")
 
         reindexed = mgr.reindex_axis(["g", "c", "a", "d"], axis=0)
-        assert reindexed.nblocks == 2
+        # reindex_axis does not consolidate_inplace, as that risks failing to
+        #  invalidate _item_cache
+        assert not reindexed.is_consolidated()
+
         tm.assert_index_equal(reindexed.items, Index(["g", "c", "a", "d"]))
         tm.assert_almost_equal(
             mgr.iget(6).internal_values(), reindexed.iget(0).internal_values()