From d52e6999c54c984c962b68a1c36c81f5f5a785d5 Mon Sep 17 00:00:00 2001 From: jreback Date: Thu, 13 Jun 2013 18:44:49 -0400 Subject: [PATCH 1/6] TST: json tests to int64 to avoid dtype issues --- pandas/io/json.py | 19 +++++++++++++++++++ pandas/io/tests/test_json/test_pandas.py | 7 +++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pandas/io/json.py b/pandas/io/json.py index fcecb31bb77a7..ce95c3394ce2c 100644 --- a/pandas/io/json.py +++ b/pandas/io/json.py @@ -268,6 +268,15 @@ def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True): except: pass + if data.dtype == 'float': + + # coerce floats to 64 + try: + data = data.astype('float64') + result = True + except: + pass + # do't coerce 0-len data if len(data) and (data.dtype == 'float' or data.dtype == 'object'): @@ -280,6 +289,16 @@ def _try_convert_data(self, name, data, use_dtypes=True, convert_dates=True): except: pass + # coerce ints to 64 + if data.dtype == 'int': + + # coerce floats to 64 + try: + data = data.astype('int64') + result = True + except: + pass + return data, result def _try_convert_to_date(self, data): diff --git a/pandas/io/tests/test_json/test_pandas.py b/pandas/io/tests/test_json/test_pandas.py index bdd700bdbcec3..fe717f56e6bea 100644 --- a/pandas/io/tests/test_json/test_pandas.py +++ b/pandas/io/tests/test_json/test_pandas.py @@ -26,7 +26,7 @@ _frame = DataFrame(_seriesd) _frame2 = DataFrame(_seriesd, columns=['D', 'C', 'B', 'A']) -_intframe = DataFrame(dict((k, v.astype(int)) +_intframe = DataFrame(dict((k, v.astype(np.int64)) for k, v in _seriesd.iteritems())) _tsframe = DataFrame(_tsd) @@ -71,6 +71,9 @@ def _check_orient(df, orient, dtype=None, numpy=False, convert_axes=True, check_ unser = unser.sort() + if dtype is False: + check_dtype=False + if not convert_axes and df.index.dtype.type == np.datetime64: unser.index = DatetimeIndex(unser.index.values.astype('i8')) if orient == "records": @@ -288,7 +291,7 @@ def test_series_to_json_except(self): def test_typ(self): - s = Series(range(6), index=['a','b','c','d','e','f']) + s = Series(range(6), index=['a','b','c','d','e','f'], dtype='int64') result = read_json(s.to_json(),typ=None) assert_series_equal(result,s) From 37fce3c6e61e596f3ee49e98d75c9d1051fa0dc7 Mon Sep 17 00:00:00 2001 From: jreback Date: Thu, 13 Jun 2013 19:06:39 -0400 Subject: [PATCH 2/6] TST: skip tests if xlrd has lower than needed version --- pandas/io/tests/test_excel.py | 6 +----- pandas/tests/test_panel.py | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/io/tests/test_excel.py b/pandas/io/tests/test_excel.py index 39e1042d125a2..baa4f6b64ec0e 100644 --- a/pandas/io/tests/test_excel.py +++ b/pandas/io/tests/test_excel.py @@ -808,11 +808,7 @@ def test_to_excel_styleconverter(self): # self.assertTrue(ws.cell(maddr).merged) # os.remove(filename) def test_excel_010_hemstring(self): - try: - import xlwt - import openpyxl - except ImportError: - raise nose.SkipTest + _skip_if_no_excelsuite() from pandas.util.testing import makeCustomDataframe as mkdf # ensure limited functionality in 0.10 diff --git a/pandas/tests/test_panel.py b/pandas/tests/test_panel.py index 380604b0de32e..4e57977a787f2 100644 --- a/pandas/tests/test_panel.py +++ b/pandas/tests/test_panel.py @@ -1381,7 +1381,11 @@ def test_to_excel(self): path = '__tmp__.' + ext with ensure_clean(path) as path: self.panel.to_excel(path) - reader = ExcelFile(path) + try: + reader = ExcelFile(path) + except ImportError: + raise nose.SkipTest + for item, df in self.panel.iterkv(): recdf = reader.parse(str(item), index_col=0) assert_frame_equal(df, recdf) From 95dfba48e5490524b17ca1613237225d7a07a289 Mon Sep 17 00:00:00 2001 From: jreback Date: Thu, 13 Jun 2013 19:12:30 -0400 Subject: [PATCH 3/6] TST: skip pickle tests on not-little endianess --- pandas/io/tests/test_pickle.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/io/tests/test_pickle.py b/pandas/io/tests/test_pickle.py index d49597860cd16..f58ef3c6919ee 100644 --- a/pandas/io/tests/test_pickle.py +++ b/pandas/io/tests/test_pickle.py @@ -8,6 +8,7 @@ import unittest import nose import os +import sys import numpy as np import pandas.util.testing as tm @@ -16,6 +17,9 @@ from pandas.sparse.tests import test_sparse from pandas.util import py3compat +if sys.byteorder != 'little': + raise nose.SkipTest('system byteorder is not little!') + class TestPickle(unittest.TestCase): _multiprocess_can_split_ = True From 33922f80165bb960f09f4c39b484371877d9358e Mon Sep 17 00:00:00 2001 From: jreback Date: Thu, 13 Jun 2013 19:16:29 -0400 Subject: [PATCH 4/6] TST: skip test_encoding on non-little endian in test_pytables --- pandas/io/tests/test_pytables.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py index 8b3d4a475d952..3266a906dcfae 100644 --- a/pandas/io/tests/test_pytables.py +++ b/pandas/io/tests/test_pytables.py @@ -477,6 +477,9 @@ def test_append(self): def test_encoding(self): + if sys.byteorder != 'little': + raise nose.SkipTest('system byteorder is not little, skipping test_encoding!') + with ensure_clean(self.path) as store: df = DataFrame(dict(A='foo',B='bar'),index=range(5)) df.loc[2,'A'] = np.nan From f6949a87d588e6c2568d147247c85c70778bdcef Mon Sep 17 00:00:00 2001 From: jreback Date: Thu, 13 Jun 2013 19:23:03 -0400 Subject: [PATCH 5/6] TST: skip some stata tests on non-little endian --- pandas/io/tests/test_stata.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/io/tests/test_stata.py b/pandas/io/tests/test_stata.py index d512b0267ed13..54875e62de218 100644 --- a/pandas/io/tests/test_stata.py +++ b/pandas/io/tests/test_stata.py @@ -3,7 +3,7 @@ from datetime import datetime import os import unittest - +import sys import warnings import nose @@ -15,6 +15,10 @@ import pandas.util.testing as tm from pandas.util.testing import ensure_clean +def _skip_if_not_little(name): + if sys.byteorder != 'little': + raise nose.SkipTest('system byteorder is not little, skipping %s' % name) + class StataTests(unittest.TestCase): def setUp(self): @@ -128,6 +132,7 @@ def test_read_dta4(self): tm.assert_frame_equal(parsed, expected) def test_write_dta5(self): + _skip_if_not_little('write_dta5') original = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)], columns=['float_miss', 'double_miss', 'byte_miss', 'int_miss', 'long_miss']) original.index.name = 'index' @@ -138,6 +143,7 @@ def test_write_dta5(self): tm.assert_frame_equal(written_and_read_again.set_index('index'), original) def test_write_dta6(self): + _skip_if_not_little('write_dta6') original = self.read_csv(self.csv3) original.index.name = 'index' From 265fabf98eba3fc1d73278d3a5680f52ed8633b6 Mon Sep 17 00:00:00 2001 From: jreback Date: Fri, 14 Jun 2013 08:41:18 -0400 Subject: [PATCH 6/6] TST: convered skips on knownfailures to knownfailures (test_pickle/test_stata) --- pandas/io/tests/test_pickle.py | 8 ++++---- pandas/io/tests/test_stata.py | 10 ++++------ pandas/util/misc.py | 7 +++++++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pandas/io/tests/test_pickle.py b/pandas/io/tests/test_pickle.py index f58ef3c6919ee..a7f0e3d3e37b1 100644 --- a/pandas/io/tests/test_pickle.py +++ b/pandas/io/tests/test_pickle.py @@ -8,7 +8,6 @@ import unittest import nose import os -import sys import numpy as np import pandas.util.testing as tm @@ -16,9 +15,8 @@ from pandas import Index from pandas.sparse.tests import test_sparse from pandas.util import py3compat - -if sys.byteorder != 'little': - raise nose.SkipTest('system byteorder is not little!') +from pandas.util.decorators import knownfailureif +from pandas.util.misc import is_little_endian class TestPickle(unittest.TestCase): _multiprocess_can_split_ = True @@ -60,6 +58,7 @@ def compare(self, vf): comparator = getattr(tm,"assert_%s_equal" % typ) comparator(result,expected) + @knownfailureif(not is_little_endian(), "known failure of test_read_pickles_0_10_1 on non-little endian") def test_read_pickles_0_10_1(self): pth = tm.get_data_path('legacy_pickle/0.10.1') @@ -67,6 +66,7 @@ def test_read_pickles_0_10_1(self): vf = os.path.join(pth,f) self.compare(vf) + @knownfailureif(not is_little_endian(), "known failure of test_read_pickles_0_11_0 on non-little endian") def test_read_pickles_0_11_0(self): pth = tm.get_data_path('legacy_pickle/0.11.0') diff --git a/pandas/io/tests/test_stata.py b/pandas/io/tests/test_stata.py index 54875e62de218..794d303a68d79 100644 --- a/pandas/io/tests/test_stata.py +++ b/pandas/io/tests/test_stata.py @@ -14,10 +14,8 @@ from pandas.io.stata import read_stata, StataReader, StataWriter import pandas.util.testing as tm from pandas.util.testing import ensure_clean - -def _skip_if_not_little(name): - if sys.byteorder != 'little': - raise nose.SkipTest('system byteorder is not little, skipping %s' % name) +from pandas.util.decorators import knownfailureif +from pandas.util.misc import is_little_endian class StataTests(unittest.TestCase): @@ -131,8 +129,8 @@ def test_read_dta4(self): tm.assert_frame_equal(parsed, expected) + @knownfailureif(not is_little_endian(), "known failure of test_write_dta5 on non-little endian") def test_write_dta5(self): - _skip_if_not_little('write_dta5') original = DataFrame([(np.nan, np.nan, np.nan, np.nan, np.nan)], columns=['float_miss', 'double_miss', 'byte_miss', 'int_miss', 'long_miss']) original.index.name = 'index' @@ -142,8 +140,8 @@ def test_write_dta5(self): written_and_read_again = self.read_dta(path) tm.assert_frame_equal(written_and_read_again.set_index('index'), original) + @knownfailureif(not is_little_endian(), "known failure of test_write_dta6 on non-little endian") def test_write_dta6(self): - _skip_if_not_little('write_dta6') original = self.read_csv(self.csv3) original.index.name = 'index' diff --git a/pandas/util/misc.py b/pandas/util/misc.py index 8372ba56d00cd..15492cde5a9f7 100644 --- a/pandas/util/misc.py +++ b/pandas/util/misc.py @@ -1,3 +1,10 @@ +""" various miscellaneous utilities """ + +def is_little_endian(): + """ am I little endian """ + import sys + return sys.byteorder == 'little' + def exclusive(*args): count = sum([arg is not None for arg in args]) return count == 1