Unstack throws when run on subset of DataFrame

[rbu]

Code Sample

Consider the following three cases:

df1 = pd.DataFrame([
        [1, 1, None, None, 30.0, None],
        [2, 2, None, None, 30.0, None],
    ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3', u'col4',]).set_index([u'ix1', 'ix2'])
df1.iloc[1:2].unstack('ix2')

df2 = pd.DataFrame([
        [1, 1, None, None, 30.0],
        [2, 2, None, None, 30.0],
    ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3']).set_index([u'ix1', 'ix2'])
df2.iloc[1:2].unstack('ix2')

df3 = pd.DataFrame([
        [1, 1, None, None, 30.0],
        [2, None, None, None, 30.0],
    ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3']).set_index([u'ix1', 'ix2'])
df3.iloc[1:2].unstack('ix2')

Problem description

When DataFrame().unstack() is run on a subset of the DataFrame (i.e. the Index levels contain values that are not present in the subset), spurious errors are triggered.

We have encountered the exception triggered by df1 in production when groupby(A).apply(B) which performed an unstack in the B function. When trying to build a reduced test case, we encountered the issues of df2 and df3 and found them worthwhile to report.

Note that none of these fail when the DataFrame has only 3 columns or less.

Expected Output

In all cases, I expect a pivoted DataFrame simila to this:

     col1  col2  col3  col4      
ix2     2     2     2     2
ix1                                                
2    None   None  30.0  None

Actual Output

# df1
IndexError                                Traceback (most recent call last)
<ipython-input-9-4c1b8dc12e05> in <module>()
      3         [2, 2, None, None, 30.0, None],
      4     ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3', u'col4',]).set_index([u'ix1', 'ix2'])
----> 5 df1.iloc[1:2].unstack('ix2')

pandas/core/frame.pyc in unstack(self, level, fill_value)
   4567         """
   4568         from pandas.core.reshape.reshape import unstack
-> 4569         return unstack(self, level, fill_value)
   4570 
   4571     _shared_docs['melt'] = ("""

pandas/core/reshape/reshape.pyc in unstack(obj, level, fill_value)
    467     if isinstance(obj, DataFrame):
    468         if isinstance(obj.index, MultiIndex):
--> 469             return _unstack_frame(obj, level, fill_value=fill_value)
    470         else:
    471             return obj.T.stack(dropna=False)

pandas/core/reshape/reshape.pyc in _unstack_frame(obj, level, fill_value)
    480         unstacker = partial(_Unstacker, index=obj.index,
    481                             level=level, fill_value=fill_value)
--> 482         blocks = obj._data.unstack(unstacker)
    483         klass = type(obj)
    484         return klass(blocks)

pandas/core/internals.pyc in unstack(self, unstacker_func)
   4349         new_columns = new_columns[columns_mask]
   4350 
-> 4351         bm = BlockManager(new_blocks, [new_columns, new_index])
   4352         return bm
   4353 

pandas/core/internals.pyc in __init__(self, blocks, axes, do_integrity_check, fastpath)
   3035         self._consolidate_check()
   3036 
-> 3037         self._rebuild_blknos_and_blklocs()
   3038 
   3039     def make_empty(self, axes=None):

pandas/core/internals.pyc in _rebuild_blknos_and_blklocs(self)
   3123         for blkno, blk in enumerate(self.blocks):
   3124             rl = blk.mgr_locs
-> 3125             new_blknos[rl.indexer] = blkno
   3126             new_blklocs[rl.indexer] = np.arange(len(rl))
   3127 

IndexError: index 7 is out of bounds for axis 1 with size 4


# df2
ValueError                                Traceback (most recent call last)
<ipython-input-4-b8998add9f5d> in <module>()
      3         [2, 2, None, None, 30.0],
      4     ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3']).set_index([u'ix1', 'ix2'])
----> 5 df2.iloc[1:2].unstack('ix2')

pandas/core/frame.pyc in unstack(self, level, fill_value)
   4567         """
   4568         from pandas.core.reshape.reshape import unstack
-> 4569         return unstack(self, level, fill_value)
   4570 
   4571     _shared_docs['melt'] = ("""

pandas/core/reshape/reshape.pyc in unstack(obj, level, fill_value)
    467     if isinstance(obj, DataFrame):
    468         if isinstance(obj.index, MultiIndex):
--> 469             return _unstack_frame(obj, level, fill_value=fill_value)
    470         else:
    471             return obj.T.stack(dropna=False)

pandas/core/reshape/reshape.pyc in _unstack_frame(obj, level, fill_value)
    480         unstacker = partial(_Unstacker, index=obj.index,
    481                             level=level, fill_value=fill_value)
--> 482         blocks = obj._data.unstack(unstacker)
    483         klass = type(obj)
    484         return klass(blocks)

pandas/core/internals.pyc in unstack(self, unstacker_func)
   4349         new_columns = new_columns[columns_mask]
   4350 
-> 4351         bm = BlockManager(new_blocks, [new_columns, new_index])
   4352         return bm
   4353 

pandas/core/internals.pyc in __init__(self, blocks, axes, do_integrity_check, fastpath)
   3035         self._consolidate_check()
   3036 
-> 3037         self._rebuild_blknos_and_blklocs()
   3038 
   3039     def make_empty(self, axes=None):

pandas/core/internals.pyc in _rebuild_blknos_and_blklocs(self)
   3124             rl = blk.mgr_locs
   3125             new_blknos[rl.indexer] = blkno
-> 3126             new_blklocs[rl.indexer] = np.arange(len(rl))
   3127 
   3128         if (new_blknos == -1).any():

ValueError: could not broadcast input array from shape (2) into shape (1)



# df3
AssertionError                            Traceback (most recent call last)
<ipython-input-5-20d7feb391ae> in <module>()
      3         [2, None, None, None, 30.0],
      4     ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3']).set_index([u'ix1', 'ix2'])
----> 5 df3.iloc[1:2].unstack('ix2')

pandas/core/frame.pyc in unstack(self, level, fill_value)
   4567         """
   4568         from pandas.core.reshape.reshape import unstack
-> 4569         return unstack(self, level, fill_value)
   4570 
   4571     _shared_docs['melt'] = ("""

pandas/core/reshape/reshape.pyc in unstack(obj, level, fill_value)
    467     if isinstance(obj, DataFrame):
    468         if isinstance(obj.index, MultiIndex):
--> 469             return _unstack_frame(obj, level, fill_value=fill_value)
    470         else:
    471             return obj.T.stack(dropna=False)

pandas/core/reshape/reshape.pyc in _unstack_frame(obj, level, fill_value)
    480         unstacker = partial(_Unstacker, index=obj.index,
    481                             level=level, fill_value=fill_value)
--> 482         blocks = obj._data.unstack(unstacker)
    483         klass = type(obj)
    484         return klass(blocks)

pandas/core/internals.pyc in unstack(self, unstacker_func)
   4349         new_columns = new_columns[columns_mask]
   4350 
-> 4351         bm = BlockManager(new_blocks, [new_columns, new_index])
   4352         return bm
   4353 

pandas/core/internals.pyc in __init__(self, blocks, axes, do_integrity_check, fastpath)
   3035         self._consolidate_check()
   3036 
-> 3037         self._rebuild_blknos_and_blklocs()
   3038 
   3039     def make_empty(self, axes=None):

pandas/core/internals.pyc in _rebuild_blknos_and_blklocs(self)
   3127 
   3128         if (new_blknos == -1).any():
-> 3129             raise AssertionError("Gaps in blk ref_locs")
   3130 
   3131         self._blknos = new_blknos

AssertionError: Gaps in blk ref_locs

Output of pd.show_versions()

INSTALLED VERSIONS
------------------
commit: None
python: 2.7.14.final.0
python-bits: 64
OS: Linux
OS-release: 4.14.13-300.fc27.x86_64
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
LOCALE: None.None

pandas: 0.22.0
pytest: None
pip: 9.0.1
setuptools: 38.2.5
Cython: None
numpy: 1.14.0
scipy: 1.0.0
pyarrow: None
xarray: None
IPython: 5.5.0
sphinx: None
patsy: None
dateutil: 2.6.1
pytz: 2017.3
blosc: None
bottleneck: 1.2.1
tables: None
numexpr: 2.6.4
feather: None
matplotlib: 2.1.2
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: 4.6.0
html5lib: 1.0.1
sqlalchemy: 0.9.10
pymysql: 0.8.0
psycopg2: None
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

[rbu]

As a workaround, this yields the expected results:

index_cols = df1.index.names
df1.iloc[1:2].reset_index().set_index(index_cols).unstack('ix2')

[toobaz]

Related to #18562 - apparently #17845 didn't fix all cases.

[jreback]

PR welcome!

[EverydayQA]

cannot repeat the error

bpython version 0.17.1 on top of Python 3.6.5 /usr/bin/python3
>>> import pandas as pd
>>> pd.__version__
'0.20.1'
>>> df3 = pd.DataFrame([
...         [1, 1, None, None, 30.0],
...         [2, None, None, None, 30.0],
...     ], columns=[u'ix1', u'ix2', u'col1', u'col2', u'col3']).set_index([u'ix1', 'ix2'])
>>>     
>>> df3.iloc[1:2].unstack('ix2')
     col1  col2  col3
ix2   NaN   NaN   NaN
ix1                  
2    None  None  30.0

[toobaz]

@EverydayQA cool, thanks!

(But surprising: before closing, I think it's worth trying to git bisect until we understand what fixed the problem)

[acarl005]

Unit tests for this issue in PR #29438