header.py

# -*- coding: utf-8 -*-

"""
Tests that the file header is properly handled or inferred
during parsing for all of the parsers defined in parsers.py
"""

from collections import namedtuple

import numpy as np
import pytest

from pandas.compat import StringIO, lrange, u
from pandas.errors import ParserError

from pandas import DataFrame, Index, MultiIndex
import pandas.util.testing as tm


class HeaderTests(object):

    def test_read_with_bad_header(self):
        errmsg = r"but only \d+ lines in file"

        with pytest.raises(ValueError, match=errmsg):
            s = StringIO(',,')
            self.read_csv(s, header=[10])

    def test_bool_header_arg(self):
        # see gh-6114
        data = """\
MyColumn
   a
   b
   a
   b"""
        for arg in [True, False]:
            with pytest.raises(TypeError):
                self.read_csv(StringIO(data), header=arg)
            with pytest.raises(TypeError):
                self.read_table(StringIO(data), header=arg)

    def test_no_header_prefix(self):
        data = """1,2,3,4,5
6,7,8,9,10
11,12,13,14,15
"""
        df_pref = self.read_table(StringIO(data), sep=',', prefix='Field',
                                  header=None)

        expected = np.array([[1, 2, 3, 4, 5],
                             [6, 7, 8, 9, 10],
                             [11, 12, 13, 14, 15]], dtype=np.int64)
        tm.assert_almost_equal(df_pref.values, expected)

        tm.assert_index_equal(df_pref.columns,
                              Index(['Field0', 'Field1', 'Field2',
                                     'Field3', 'Field4']))

    def test_header_with_index_col(self):
        data = """foo,1,2,3
bar,4,5,6
baz,7,8,9
"""
        names = ['A', 'B', 'C']
        df = self.read_csv(StringIO(data), names=names)

        assert list(df.columns) == ['A', 'B', 'C']

        values = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
        expected = DataFrame(values, index=['foo', 'bar', 'baz'],
                             columns=['A', 'B', 'C'])
        tm.assert_frame_equal(df, expected)

    def test_header_not_first_line(self):
        data = """got,to,ignore,this,line
got,to,ignore,this,line
index,A,B,C,D
foo,2,3,4,5
bar,7,8,9,10
baz,12,13,14,15
"""
        data2 = """index,A,B,C,D
foo,2,3,4,5
bar,7,8,9,10
baz,12,13,14,15
"""

        df = self.read_csv(StringIO(data), header=2, index_col=0)
        expected = self.read_csv(StringIO(data2), header=0, index_col=0)
        tm.assert_frame_equal(df, expected)

    def test_header_multi_index(self):
        expected = tm.makeCustomDataframe(
            5, 3, r_idx_nlevels=2, c_idx_nlevels=4)

        data = """\
C0,,C_l0_g0,C_l0_g1,C_l0_g2

C1,,C_l1_g0,C_l1_g1,C_l1_g2
C2,,C_l2_g0,C_l2_g1,C_l2_g2
C3,,C_l3_g0,C_l3_g1,C_l3_g2
R0,R1,,,
R_l0_g0,R_l1_g0,R0C0,R0C1,R0C2
R_l0_g1,R_l1_g1,R1C0,R1C1,R1C2
R_l0_g2,R_l1_g2,R2C0,R2C1,R2C2
R_l0_g3,R_l1_g3,R3C0,R3C1,R3C2
R_l0_g4,R_l1_g4,R4C0,R4C1,R4C2
"""

        df = self.read_csv(StringIO(data), header=[0, 1, 2, 3],
                           index_col=[0, 1])
        tm.assert_frame_equal(df, expected)

        # skipping lines in the header
        df = self.read_csv(StringIO(data), header=[0, 1, 2, 3],
                           index_col=[0, 1])
        tm.assert_frame_equal(df, expected)

        # INVALID OPTIONS

        # names
        pytest.raises(ValueError, self.read_csv,
                      StringIO(data), header=[0, 1, 2, 3],
                      index_col=[0, 1], names=['foo', 'bar'])

        # usecols
        pytest.raises(ValueError, self.read_csv,
                      StringIO(data), header=[0, 1, 2, 3],
                      index_col=[0, 1], usecols=['foo', 'bar'])

        # non-numeric index_col
        pytest.raises(ValueError, self.read_csv,
                      StringIO(data), header=[0, 1, 2, 3],
                      index_col=['foo', 'bar'])

    def test_header_multiindex_common_format(self):

        df = DataFrame([[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]],
                       index=['one', 'two'],
                       columns=MultiIndex.from_tuples(
                           [('a', 'q'), ('a', 'r'), ('a', 's'),
                            ('b', 't'), ('c', 'u'), ('c', 'v')]))

        # to_csv
        data = """,a,a,a,b,c,c
,q,r,s,t,u,v
,,,,,,
one,1,2,3,4,5,6
two,7,8,9,10,11,12"""

        result = self.read_csv(StringIO(data), header=[0, 1], index_col=0)
        tm.assert_frame_equal(df, result)

        # to_csv, tuples
        result = self.read_csv(StringIO(data), skiprows=3,
                               names=[('a', 'q'), ('a', 'r'), ('a', 's'),
                                      ('b', 't'), ('c', 'u'), ('c', 'v')],
                               index_col=0)
        tm.assert_frame_equal(df, result)

        # to_csv, namedtuples
        TestTuple = namedtuple('names', ['first', 'second'])
        result = self.read_csv(
            StringIO(data), skiprows=3, index_col=0,
            names=[TestTuple('a', 'q'), TestTuple('a', 'r'),
                   TestTuple('a', 's'), TestTuple('b', 't'),
                   TestTuple('c', 'u'), TestTuple('c', 'v')])
        tm.assert_frame_equal(df, result)

        # common
        data = """,a,a,a,b,c,c
,q,r,s,t,u,v
one,1,2,3,4,5,6
two,7,8,9,10,11,12"""

        result = self.read_csv(StringIO(data), header=[0, 1], index_col=0)
        tm.assert_frame_equal(df, result)

        # common, tuples
        result = self.read_csv(StringIO(data), skiprows=2,
                               names=[('a', 'q'), ('a', 'r'), ('a', 's'),
                                      ('b', 't'), ('c', 'u'), ('c', 'v')],
                               index_col=0)
        tm.assert_frame_equal(df, result)

        # common, namedtuples
        TestTuple = namedtuple('names', ['first', 'second'])
        result = self.read_csv(
            StringIO(data), skiprows=2, index_col=0,
            names=[TestTuple('a', 'q'), TestTuple('a', 'r'),
                   TestTuple('a', 's'), TestTuple('b', 't'),
                   TestTuple('c', 'u'), TestTuple('c', 'v')])
        tm.assert_frame_equal(df, result)

        # common, no index_col
        data = """a,a,a,b,c,c
q,r,s,t,u,v
1,2,3,4,5,6
7,8,9,10,11,12"""

        result = self.read_csv(StringIO(data), header=[0, 1], index_col=None)
        tm.assert_frame_equal(df.reset_index(drop=True), result)

        # common, no index_col, tuples
        result = self.read_csv(StringIO(data), skiprows=2,
                               names=[('a', 'q'), ('a', 'r'), ('a', 's'),
                                      ('b', 't'), ('c', 'u'), ('c', 'v')],
                               index_col=None)
        tm.assert_frame_equal(df.reset_index(drop=True), result)

        # common, no index_col, namedtuples
        TestTuple = namedtuple('names', ['first', 'second'])
        result = self.read_csv(
            StringIO(data), skiprows=2, index_col=None,
            names=[TestTuple('a', 'q'), TestTuple('a', 'r'),
                   TestTuple('a', 's'), TestTuple('b', 't'),
                   TestTuple('c', 'u'), TestTuple('c', 'v')])
        tm.assert_frame_equal(df.reset_index(drop=True), result)

        # malformed case 1
        expected = DataFrame(np.array(
            [[2, 3, 4, 5, 6], [8, 9, 10, 11, 12]], dtype='int64'),
            index=Index([1, 7]),
            columns=MultiIndex(levels=[[u('a'), u('b'), u('c')],
                                       [u('r'), u('s'), u('t'),
                                        u('u'), u('v')]],
                               labels=[[0, 0, 1, 2, 2], [0, 1, 2, 3, 4]],
                               names=[u('a'), u('q')]))

        data = """a,a,a,b,c,c
q,r,s,t,u,v
1,2,3,4,5,6
7,8,9,10,11,12"""

        result = self.read_csv(StringIO(data), header=[0, 1], index_col=0)
        tm.assert_frame_equal(expected, result)

        # malformed case 2
        expected = DataFrame(np.array(
            [[2, 3, 4, 5, 6], [8, 9, 10, 11, 12]], dtype='int64'),
            index=Index([1, 7]),
            columns=MultiIndex(levels=[[u('a'), u('b'), u('c')],
                                       [u('r'), u('s'), u('t'),
                                        u('u'), u('v')]],
                               labels=[[0, 0, 1, 2, 2], [0, 1, 2, 3, 4]],
                               names=[None, u('q')]))

        data = """,a,a,b,c,c
q,r,s,t,u,v
1,2,3,4,5,6
7,8,9,10,11,12"""

        result = self.read_csv(StringIO(data), header=[0, 1], index_col=0)
        tm.assert_frame_equal(expected, result)

        # mi on columns and index (malformed)
        expected = DataFrame(np.array(
            [[3, 4, 5, 6], [9, 10, 11, 12]], dtype='int64'),
            index=MultiIndex(levels=[[1, 7], [2, 8]],
                             labels=[[0, 1], [0, 1]]),
            columns=MultiIndex(levels=[[u('a'), u('b'), u('c')],
                                       [u('s'), u('t'), u('u'), u('v')]],
                               labels=[[0, 1, 2, 2], [0, 1, 2, 3]],
                               names=[None, u('q')]))

        data = """,a,a,b,c,c
q,r,s,t,u,v
1,2,3,4,5,6
7,8,9,10,11,12"""

        result = self.read_csv(StringIO(data), header=[0, 1], index_col=[0, 1])
        tm.assert_frame_equal(expected, result)

    def test_header_names_backward_compat(self):
        # #2539
        data = '1,2,3\n4,5,6'

        result = self.read_csv(StringIO(data), names=['a', 'b', 'c'])
        expected = self.read_csv(StringIO(data), names=['a', 'b', 'c'],
                                 header=None)
        tm.assert_frame_equal(result, expected)

        data2 = 'foo,bar,baz\n' + data
        result = self.read_csv(StringIO(data2), names=['a', 'b', 'c'],
                               header=0)
        tm.assert_frame_equal(result, expected)

    def test_read_only_header_no_rows(self):
        # See gh-7773
        expected = DataFrame(columns=['a', 'b', 'c'])

        df = self.read_csv(StringIO('a,b,c'))
        tm.assert_frame_equal(df, expected)

        df = self.read_csv(StringIO('a,b,c'), index_col=False)
        tm.assert_frame_equal(df, expected)

    def test_no_header(self):
        data = """1,2,3,4,5
6,7,8,9,10
11,12,13,14,15
"""
        df = self.read_table(StringIO(data), sep=',', header=None)
        df_pref = self.read_table(StringIO(data), sep=',', prefix='X',
                                  header=None)

        names = ['foo', 'bar', 'baz', 'quux', 'panda']
        df2 = self.read_table(StringIO(data), sep=',', names=names)
        expected = np.array([[1, 2, 3, 4, 5],
                             [6, 7, 8, 9, 10],
                             [11, 12, 13, 14, 15]], dtype=np.int64)
        tm.assert_almost_equal(df.values, expected)
        tm.assert_almost_equal(df.values, df2.values)

        tm.assert_index_equal(df_pref.columns,
                              Index(['X0', 'X1', 'X2', 'X3', 'X4']))
        tm.assert_index_equal(df.columns, Index(lrange(5)))

        tm.assert_index_equal(df2.columns, Index(names))

    def test_non_int_header(self):
        # GH 16338
        msg = 'header must be integer or list of integers'
        data = """1,2\n3,4"""
        with pytest.raises(ValueError, match=msg):
            self.read_csv(StringIO(data), sep=',', header=['a', 'b'])
        with pytest.raises(ValueError, match=msg):
            self.read_csv(StringIO(data), sep=',', header='string_header')

    def test_singleton_header(self):
        # See GH #7757
        data = """a,b,c\n0,1,2\n1,2,3"""
        df = self.read_csv(StringIO(data), header=[0])
        expected = DataFrame({"a": [0, 1], "b": [1, 2], "c": [2, 3]})
        tm.assert_frame_equal(df, expected)

    def test_mangles_multi_index(self):
        # See GH 18062
        data = """A,A,A,B\none,one,one,two\n0,40,34,0.1"""
        df = self.read_csv(StringIO(data), header=[0, 1])
        expected = DataFrame([[0, 40, 34, 0.1]],
                             columns=MultiIndex.from_tuples(
                                 [('A', 'one'), ('A', 'one.1'),
                                  ('A', 'one.2'), ('B', 'two')]))
        tm.assert_frame_equal(df, expected)

        data = """A,A,A,B\none,one,one.1,two\n0,40,34,0.1"""
        df = self.read_csv(StringIO(data), header=[0, 1])
        expected = DataFrame([[0, 40, 34, 0.1]],
                             columns=MultiIndex.from_tuples(
                                 [('A', 'one'), ('A', 'one.1'),
                                  ('A', 'one.1.1'), ('B', 'two')]))
        tm.assert_frame_equal(df, expected)

        data = """A,A,A,B,B\none,one,one.1,two,two\n0,40,34,0.1,0.1"""
        df = self.read_csv(StringIO(data), header=[0, 1])
        expected = DataFrame([[0, 40, 34, 0.1, 0.1]],
                             columns=MultiIndex.from_tuples(
                                 [('A', 'one'), ('A', 'one.1'),
                                  ('A', 'one.1.1'), ('B', 'two'),
                                  ('B', 'two.1')]))
        tm.assert_frame_equal(df, expected)

    @pytest.mark.parametrize("index_col", [None, [0]])
    @pytest.mark.parametrize("columns", [None,
                                         (["", "Unnamed"]),
                                         (["Unnamed", ""]),
                                         (["Unnamed", "NotUnnamed"])])
    def test_multi_index_unnamed(self, index_col, columns):
        # see gh-23687
        #
        # When specifying a multi-index header, make sure that
        # we don't error just because one of the rows in our header
        # has ALL column names containing the string "Unnamed". The
        # correct condition to check is whether the row contains
        # ALL columns that did not have names (and instead were given
        # placeholder ones).
        header = [0, 1]

        if index_col is None:
            data = ",".join(columns or ["", ""]) + "\n0,1\n2,3\n4,5\n"
        else:
            data = (",".join([""] + (columns or ["", ""])) +
                    "\n,0,1\n0,2,3\n1,4,5\n")

        if columns is None:
            msg = (r"Passed header=\[0,1\] are too "
                   r"many rows for this multi_index of columns")
            with pytest.raises(ParserError, match=msg):
                self.read_csv(StringIO(data), header=header,
                              index_col=index_col)
        else:
            result = self.read_csv(StringIO(data), header=header,
                                   index_col=index_col)
            template = "Unnamed: {i}_level_0"
            exp_columns = []

            for i, col in enumerate(columns):
                if not col:  # Unnamed.
                    col = template.format(i=i if index_col is None else i + 1)

                exp_columns.append(col)

            columns = MultiIndex.from_tuples(zip(exp_columns, ["0", "1"]))
            expected = DataFrame([[2, 3], [4, 5]], columns=columns)
            tm.assert_frame_equal(result, expected)