From d07ae989104ac1bafbb30ef3730f259ec84c52fd Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Fri, 1 Nov 2013 20:30:47 -0400 Subject: [PATCH 1/2] TST: Use tempfiles in all tests. Includes @jreback's commits from #5422 and hdf_temp: * TST: make pytables tests go thru a temporary dir and file * TST/BUG: incorrect way of testing for r+ modes TST: fix temporary files by using mktemp (rather than mkstemp) which opens them --- doc/source/release.rst | 1 + pandas/io/pytables.py | 24 ++++-- pandas/io/tests/test_excel.py | 134 +++++++++++-------------------- pandas/io/tests/test_pytables.py | 18 ++++- pandas/tests/test_frame.py | 10 +-- pandas/util/testing.py | 26 +++--- 6 files changed, 100 insertions(+), 113 deletions(-) diff --git a/doc/source/release.rst b/doc/source/release.rst index 6e10bd651d90a..04642c358d000 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -769,6 +769,7 @@ Bug Fixes - The GroupBy methods ``transform`` and ``filter`` can be used on Series and DataFrames that have repeated (non-unique) indices. (:issue:`4620`) - Fix empty series not printing name in repr (:issue:`4651`) + - Make tests create temp files in temp directory by default. (:issue:`5419`) pandas 0.12.0 ------------- diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 97dc8dcdec73a..975d04c185d51 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -225,11 +225,6 @@ def _tables(): return _table_mod -def h5_open(path, mode): - tables = _tables() - return tables.openFile(path, mode) - - @contextmanager def get_store(path, **kwargs): """ @@ -389,6 +384,10 @@ def root(self): self._check_if_open() return self._handle.root + @property + def filename(self): + return self._path + def __getitem__(self, key): return self.get(key) @@ -475,6 +474,8 @@ def open(self, mode='a'): mode : {'a', 'w', 'r', 'r+'}, default 'a' See HDFStore docstring or tables.openFile for info about modes """ + tables = _tables() + if self._mode != mode: # if we are chaning a write mode to read, ok @@ -501,13 +502,20 @@ def open(self, mode='a'): fletcher32=self._fletcher32) try: - self._handle = h5_open(self._path, self._mode) - except IOError as e: # pragma: no cover + self._handle = tables.openFile(self._path, self._mode) + except (IOError) as e: # pragma: no cover if 'can not be written' in str(e): print('Opening %s in read-only mode' % self._path) - self._handle = h5_open(self._path, 'r') + self._handle = tables.openFile(self._path, 'r') else: raise + except (Exception) as e: + + # trying to read from a non-existant file causes an error which + # is not part of IOError, make it one + if self._mode == 'r' and 'Unable to open/create file' in str(e): + raise IOError(str(e)) + raise def close(self): """ diff --git a/pandas/io/tests/test_excel.py b/pandas/io/tests/test_excel.py index 311a0953f1c02..6eb3cbf1a3903 100644 --- a/pandas/io/tests/test_excel.py +++ b/pandas/io/tests/test_excel.py @@ -261,10 +261,9 @@ def test_read_xlrd_Book(self): import xlrd - pth = '__tmp_excel_read_worksheet__.xls' df = self.frame - with ensure_clean(pth) as pth: + with ensure_clean('.xls') as pth: df.to_excel(pth, "SheetA") book = xlrd.open_workbook(pth) @@ -303,7 +302,7 @@ def test_reader_closes_file(self): f = open(pth, 'rb') with ExcelFile(f) as xlsx: # parses okay - df = xlsx.parse('Sheet1', index_col=0) + xlsx.parse('Sheet1', index_col=0) self.assertTrue(f.closed) @@ -364,12 +363,12 @@ class ExcelWriterBase(SharedItems): # 1. A check_skip function that skips your tests if your writer isn't # installed. # 2. Add a property ext, which is the file extension that your writer - # writes to. + # writes to. (needs to start with '.' so it's a valid path) # 3. Add a property engine_name, which is the name of the writer class. def setUp(self): self.check_skip() super(ExcelWriterBase, self).setUp() - self.option_name = 'io.excel.%s.writer' % self.ext + self.option_name = 'io.excel.%s.writer' % self.ext.strip('.') self.prev_engine = get_option(self.option_name) set_option(self.option_name, self.engine_name) @@ -380,10 +379,7 @@ def test_excel_sheet_by_name_raise(self): _skip_if_no_xlrd() import xlrd - ext = self.ext - pth = os.path.join(self.dirpath, 'testit.{0}'.format(ext)) - - with ensure_clean(pth) as pth: + with ensure_clean(self.ext) as pth: gt = DataFrame(np.random.randn(10, 2)) gt.to_excel(pth) xl = ExcelFile(pth) @@ -394,10 +390,8 @@ def test_excel_sheet_by_name_raise(self): def test_excelwriter_contextmanager(self): _skip_if_no_xlrd() - ext = self.ext - pth = os.path.join(self.dirpath, 'testit.{0}'.format(ext)) - with ensure_clean(pth) as pth: + with ensure_clean(self.ext) as pth: with ExcelWriter(pth) as writer: self.frame.to_excel(writer, 'Data1') self.frame2.to_excel(writer, 'Data2') @@ -410,10 +404,8 @@ def test_excelwriter_contextmanager(self): def test_roundtrip(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.frame['A'][:5] = nan self.frame.to_excel(path, 'test1') @@ -446,10 +438,8 @@ def test_roundtrip(self): def test_mixed(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_mixed__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.mixed_frame.to_excel(path, 'test1') reader = ExcelFile(path) recons = reader.parse('test1', index_col=0) @@ -457,12 +447,10 @@ def test_mixed(self): def test_tsframe(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_tsframe__.' + ext df = tm.makeTimeDataFrame()[:5] - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: df.to_excel(path, 'test1') reader = ExcelFile(path) recons = reader.parse('test1') @@ -470,22 +458,19 @@ def test_tsframe(self): def test_basics_with_nan(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_int_types__.' + ext - self.frame['A'][:5] = nan - self.frame.to_excel(path, 'test1') - self.frame.to_excel(path, 'test1', cols=['A', 'B']) - self.frame.to_excel(path, 'test1', header=False) - self.frame.to_excel(path, 'test1', index=False) + with ensure_clean(self.ext) as path: + self.frame['A'][:5] = nan + self.frame.to_excel(path, 'test1') + self.frame.to_excel(path, 'test1', cols=['A', 'B']) + self.frame.to_excel(path, 'test1', header=False) + self.frame.to_excel(path, 'test1', index=False) def test_int_types(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_int_types__.' + ext for np_type in (np.int8, np.int16, np.int32, np.int64): - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: # Test np.int values read come back as int (rather than float # which is Excel's format). frame = DataFrame(np.random.randint(-10, 10, size=(10, 2)), @@ -505,11 +490,9 @@ def test_int_types(self): def test_float_types(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_float_types__.' + ext for np_type in (np.float16, np.float32, np.float64): - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: # Test np.float values read come back as float. frame = DataFrame(np.random.random_sample(10), dtype=np_type) frame.to_excel(path, 'test1') @@ -519,11 +502,9 @@ def test_float_types(self): def test_bool_types(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_bool_types__.' + ext for np_type in (np.bool8, np.bool_): - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: # Test np.bool values read come back as float. frame = (DataFrame([1, 0, True, False], dtype=np_type)) frame.to_excel(path, 'test1') @@ -533,10 +514,8 @@ def test_bool_types(self): def test_sheets(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_sheets__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.frame['A'][:5] = nan self.frame.to_excel(path, 'test1') @@ -560,10 +539,8 @@ def test_sheets(self): def test_colaliases(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_aliases__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.frame['A'][:5] = nan self.frame.to_excel(path, 'test1') @@ -582,10 +559,8 @@ def test_colaliases(self): def test_roundtrip_indexlabels(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_indexlabels__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.frame['A'][:5] = nan @@ -617,10 +592,7 @@ def test_roundtrip_indexlabels(self): frame.index.names = ['test'] self.assertEqual(frame.index.names, recons.index.names) - # test index_labels in same row as column names - path = '%s.%s' % (tm.rands(10), ext) - - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.frame.to_excel(path, 'test1', cols=['A', 'B', 'C', 'D'], index=False) @@ -636,12 +608,10 @@ def test_roundtrip_indexlabels(self): def test_excel_roundtrip_indexname(self): _skip_if_no_xlrd() - path = '%s.%s' % (tm.rands(10), self.ext) - df = DataFrame(np.random.randn(10, 4)) df.index.name = 'foo' - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: df.to_excel(path) xf = ExcelFile(path) @@ -656,7 +626,7 @@ def test_excel_roundtrip_datetime(self): # datetime.date, not sure what to test here exactly path = '__tmp_excel_roundtrip_datetime__.' + self.ext tsf = self.tsframe.copy() - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: tsf.index = [x.date() for x in self.tsframe.index] tsf.to_excel(path, 'test1') @@ -670,7 +640,7 @@ def test_to_excel_periodindex(self): frame = self.tsframe xp = frame.resample('M', kind='period') - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: xp.to_excel(path, 'sht1') reader = ExcelFile(path) @@ -679,8 +649,6 @@ def test_to_excel_periodindex(self): def test_to_excel_multiindex(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_multiindex__' + ext + '__.' + ext frame = self.frame old_index = frame.index @@ -689,7 +657,7 @@ def test_to_excel_multiindex(self): names=['first', 'second']) frame.index = new_index - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: frame.to_excel(path, 'test1', header=False) frame.to_excel(path, 'test1', cols=['A', 'B']) @@ -703,8 +671,6 @@ def test_to_excel_multiindex(self): def test_to_excel_multiindex_dates(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_multiindex_dates__' + ext + '__.' + ext # try multiindex with dates tsframe = self.tsframe @@ -712,7 +678,7 @@ def test_to_excel_multiindex_dates(self): new_index = [old_index, np.arange(len(old_index))] tsframe.index = MultiIndex.from_arrays(new_index) - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: tsframe.to_excel(path, 'test1', index_label=['time', 'foo']) reader = ExcelFile(path) recons = reader.parse('test1', index_col=[0, 1]) @@ -736,7 +702,7 @@ def test_to_excel_float_format(self): [12.32112, 123123.2, 321321.2]], index=['A', 'B'], columns=['X', 'Y', 'Z']) - with ensure_clean(filename) as filename: + with ensure_clean(self.ext) as filename: df.to_excel(filename, 'test1', float_format='%.2f') reader = ExcelFile(filename) @@ -748,21 +714,18 @@ def test_to_excel_float_format(self): def test_to_excel_unicode_filename(self): _skip_if_no_xlrd() - ext = self.ext - filename = u('\u0192u.') + ext - - try: - f = open(filename, 'wb') - except UnicodeEncodeError: - raise nose.SkipTest('no unicode file names on this system') - else: - f.close() - - df = DataFrame([[0.123456, 0.234567, 0.567567], - [12.32112, 123123.2, 321321.2]], - index=['A', 'B'], columns=['X', 'Y', 'Z']) + with ensure_clean(u('\u0192u.') + self.ext) as filename: + try: + f = open(filename, 'wb') + except UnicodeEncodeError: + raise nose.SkipTest('no unicode file names on this system') + else: + f.close() + + df = DataFrame([[0.123456, 0.234567, 0.567567], + [12.32112, 123123.2, 321321.2]], + index=['A', 'B'], columns=['X', 'Y', 'Z']) - with ensure_clean(filename) as filename: df.to_excel(filename, 'test1', float_format='%.2f') reader = ExcelFile(filename) @@ -879,10 +842,9 @@ def test_excel_010_hemstring(self): # override of #2370 until sorted out in 0.11 def roundtrip(df, header=True, parser_hdr=0): - path = '__tmp__test_xl_010_%s__.%s' % (np.random.randint(1, 10000), self.ext) - df.to_excel(path, header=header) - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: + df.to_excel(path, header=header) xf = pd.ExcelFile(path) res = xf.parse(xf.sheet_names[0], header=parser_hdr) return res @@ -926,10 +888,8 @@ def roundtrip(df, header=True, parser_hdr=0): def test_duplicated_columns(self): # Test for issue #5235. _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_duplicated_columns__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: write_frame = DataFrame([[1, 2, 3], [1, 2, 3], [1, 2, 3]]) colnames = ['A', 'B', 'B'] @@ -943,7 +903,7 @@ def test_duplicated_columns(self): class OpenpyxlTests(ExcelWriterBase, unittest.TestCase): - ext = 'xlsx' + ext = '.xlsx' engine_name = 'openpyxl' check_skip = staticmethod(_skip_if_no_openpyxl) @@ -974,7 +934,7 @@ def test_to_excel_styleconverter(self): class XlwtTests(ExcelWriterBase, unittest.TestCase): - ext = 'xls' + ext = '.xls' engine_name = 'xlwt' check_skip = staticmethod(_skip_if_no_xlwt) @@ -999,7 +959,7 @@ def test_to_excel_styleconverter(self): class XlsxWriterTests(ExcelWriterBase, unittest.TestCase): - ext = 'xlsx' + ext = '.xlsx' engine_name = 'xlsxwriter' check_skip = staticmethod(_skip_if_no_xlsxwriter) @@ -1007,10 +967,8 @@ class XlsxWriterTests(ExcelWriterBase, unittest.TestCase): # floating point values read back in from the output XlsxWriter file. def test_roundtrip_indexlabels(self): _skip_if_no_xlrd() - ext = self.ext - path = '__tmp_to_excel_from_excel_indexlabels__.' + ext - with ensure_clean(path) as path: + with ensure_clean(self.ext) as path: self.frame['A'][:5] = nan diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py index 598f374e0fcf7..99a77a64c3d2c 100644 --- a/pandas/io/tests/test_pytables.py +++ b/pandas/io/tests/test_pytables.py @@ -3,6 +3,7 @@ import sys import os import warnings +import tempfile from contextlib import contextmanager import datetime @@ -57,9 +58,15 @@ def safe_close(store): @contextmanager def ensure_clean(path, mode='a', complevel=None, complib=None, fletcher32=False): - store = HDFStore(path, mode=mode, complevel=complevel, - complib=complib, fletcher32=False) + try: + + # put in the temporary path if we don't have one already + if not len(os.path.dirname(path)): + path = tm.create_temp_file(path) + + store = HDFStore(path, mode=mode, complevel=complevel, + complib=complib, fletcher32=False) yield store finally: safe_close(store) @@ -94,7 +101,7 @@ class TestHDFStore(unittest.TestCase): def setUp(self): warnings.filterwarnings(action='ignore', category=FutureWarning) - self.path = '__%s__.h5' % tm.rands(10) + self.path = '.__%s__.h5' % tm.rands(10) def tearDown(self): pass @@ -192,17 +199,22 @@ def test_api(self): with ensure_clean(self.path) as store: + path = store._path df = tm.makeDataFrame() + + _maybe_remove(store,'df') store.append('df',df.iloc[:10],append=True,format='table') store.append('df',df.iloc[10:],append=True,format='table') assert_frame_equal(read_hdf(path,'df'),df) # append to False + _maybe_remove(store,'df') store.append('df',df.iloc[:10],append=False,format='table') store.append('df',df.iloc[10:],append=True,format='table') assert_frame_equal(read_hdf(path,'df'),df) # formats + _maybe_remove(store,'df') store.append('df',df.iloc[:10],append=False,format='table') store.append('df',df.iloc[10:],append=True,format='table') assert_frame_equal(read_hdf(path,'df'),df) diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py index b73c7cdbb8f87..22f5fc527d8f5 100644 --- a/pandas/tests/test_frame.py +++ b/pandas/tests/test_frame.py @@ -5477,12 +5477,13 @@ def make_dtnat_arr(n,nnat=None): # N=35000 s1=make_dtnat_arr(chunksize+5) s2=make_dtnat_arr(chunksize+5,0) + path = '1.csv' - # s3=make_dtnat_arr(chunksize+5,0) - with ensure_clean('1.csv') as path: + # s3=make_dtnjat_arr(chunksize+5,0) + with ensure_clean('.csv') as pth: df=DataFrame(dict(a=s1,b=s2)) - df.to_csv(path,chunksize=chunksize) - recons = DataFrame.from_csv(path).convert_objects('coerce') + df.to_csv(pth,chunksize=chunksize) + recons = DataFrame.from_csv(pth).convert_objects('coerce') assert_frame_equal(df, recons,check_names=False,check_less_precise=True) for ncols in [4]: @@ -5491,7 +5492,6 @@ def make_dtnat_arr(n,nnat=None): base-1,base,base+1]: _do_test(mkdf(nrows, ncols,r_idx_type='dt', c_idx_type='s'),path, 'dt','s') - pass for ncols in [4]: diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 2e4d1f3e8df74..9e742f2065419 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -313,34 +313,42 @@ def ensure_clean(filename=None, return_filelike=False): ---------- filename : str (optional) if None, creates a temporary file which is then removed when out of - scope. - return_filelike: bool (default False) + scope. if passed, creates temporary file with filename as ending. + return_filelike : bool (default False) if True, returns a file-like which is *always* cleaned. Necessary for - savefig and other functions which want to append extensions. Ignores - filename if True. + savefig and other functions which want to append extensions. """ + filename = filename or '' if return_filelike: - f = tempfile.TemporaryFile() + f = tempfile.TemporaryFile(suffix=filename) try: yield f finally: f.close() else: - # if we are not passed a filename, generate a temporary - if filename is None: - filename = tempfile.mkstemp()[1] + + # don't generate tempfile if using a path with directory specified + if len(os.path.dirname(filename)): + raise ValueError("Can't pass a qualified name to ensure_clean()") try: + filename = create_temp_file(filename) yield filename finally: try: if os.path.exists(filename): os.remove(filename) except Exception as e: - print(e) + print("Exception on removing file: %s" % e) + +def create_temp_file(filename): + """ + create a temporary file. the caller is responsible for deleting the file + """ + return tempfile.mktemp(suffix=filename) def get_data_path(f=''): """Return the path of a data file, these are relative to the current test From 9058d50f0c20b624c4079433f7cdcfd1ec92f65c Mon Sep 17 00:00:00 2001 From: jreback Date: Sun, 3 Nov 2013 19:36:33 -0500 Subject: [PATCH 2/2] TST: provide temporary file creation inside of io/pytables.py directly w/o opening a named temporary file TST: named temporary file creation back to mkstemp in utils/testing.py/ensure_clean --- pandas/io/tests/test_pytables.py | 265 ++++++++++++++++--------------- pandas/util/testing.py | 9 +- 2 files changed, 142 insertions(+), 132 deletions(-) diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py index 99a77a64c3d2c..737acef209a50 100644 --- a/pandas/io/tests/test_pytables.py +++ b/pandas/io/tests/test_pytables.py @@ -55,15 +55,19 @@ def safe_close(store): pass +def create_tempfile(path): + """ create an unopened named temporary file """ + return os.path.join(tempfile.gettempdir(),path) + @contextmanager -def ensure_clean(path, mode='a', complevel=None, complib=None, +def ensure_clean_store(path, mode='a', complevel=None, complib=None, fletcher32=False): try: # put in the temporary path if we don't have one already if not len(os.path.dirname(path)): - path = tm.create_temp_file(path) + path = create_tempfile(path) store = HDFStore(path, mode=mode, complevel=complevel, complib=complib, fletcher32=False) @@ -73,6 +77,19 @@ def ensure_clean(path, mode='a', complevel=None, complib=None, if mode == 'w' or mode == 'a': safe_remove(path) +@contextmanager +def ensure_clean_path(path): + """ + return essentially a named temporary file that is not opened + and deleted on existing + """ + + try: + filename = create_tempfile(path) + yield filename + finally: + safe_remove(filename) + # set these parameters so we don't have file sharing tables.parameters.MAX_NUMEXPR_THREADS = 1 tables.parameters.MAX_BLOSC_THREADS = 1 @@ -101,7 +118,7 @@ class TestHDFStore(unittest.TestCase): def setUp(self): warnings.filterwarnings(action='ignore', category=FutureWarning) - self.path = '.__%s__.h5' % tm.rands(10) + self.path = 'tmp.__%s__.h5' % tm.rands(10) def tearDown(self): pass @@ -158,7 +175,7 @@ def test_api(self): # GH4584 # API issue when to_hdf doesn't acdept append AND format args - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeDataFrame() df.iloc[:10].to_hdf(path,'df',append=True,format='table') @@ -170,7 +187,7 @@ def test_api(self): df.iloc[10:].to_hdf(path,'df',append=True,format='table') assert_frame_equal(read_hdf(path,'df'),df) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeDataFrame() df.iloc[:10].to_hdf(path,'df',append=True) @@ -182,7 +199,7 @@ def test_api(self): df.iloc[10:].to_hdf(path,'df',append=True) assert_frame_equal(read_hdf(path,'df'),df) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeDataFrame() df.to_hdf(path,'df',append=False,format='fixed') @@ -197,7 +214,7 @@ def test_api(self): df.to_hdf(path,'df') assert_frame_equal(read_hdf(path,'df'),df) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: path = store._path df = tm.makeDataFrame() @@ -224,7 +241,7 @@ def test_api(self): store.append('df',df.iloc[10:],append=True,format=None) assert_frame_equal(read_hdf(path,'df'),df) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: # invalid df = tm.makeDataFrame() @@ -238,7 +255,7 @@ def test_api(self): def test_api_default_format(self): # default_format option - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeDataFrame() pandas.set_option('io.hdf.default_format','fixed') @@ -257,7 +274,7 @@ def test_api_default_format(self): pandas.set_option('io.hdf.default_format',None) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeDataFrame() @@ -279,7 +296,7 @@ def test_api_default_format(self): def test_keys(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['a'] = tm.makeTimeSeries() store['b'] = tm.makeStringSeries() store['c'] = tm.makeDataFrame() @@ -291,7 +308,7 @@ def test_keys(self): def test_repr(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: repr(store) store['a'] = tm.makeTimeSeries() store['b'] = tm.makeStringSeries() @@ -326,7 +343,7 @@ def test_repr(self): str(store) # storers - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeDataFrame() store.append('df',df) @@ -337,7 +354,7 @@ def test_repr(self): def test_contains(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['a'] = tm.makeTimeSeries() store['b'] = tm.makeDataFrame() store['foo/bar'] = tm.makeDataFrame() @@ -356,7 +373,7 @@ def test_contains(self): def test_versioning(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['a'] = tm.makeTimeSeries() store['b'] = tm.makeDataFrame() df = tm.makeTimeDataFrame() @@ -382,7 +399,7 @@ def test_mode(self): def check(mode): - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: # constructor if mode in ['r','r+']: @@ -393,7 +410,7 @@ def check(mode): self.assert_(store._handle.mode == mode) store.close() - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: # context if mode in ['r','r+']: @@ -405,7 +422,7 @@ def f(): with get_store(path,mode=mode) as store: self.assert_(store._handle.mode == mode) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: # conv write if mode in ['r','r+']: @@ -428,7 +445,7 @@ def f(): def test_reopen_handle(self): - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: store = HDFStore(path,mode='a') store['a'] = tm.makeTimeSeries() @@ -474,14 +491,14 @@ def test_reopen_handle(self): def test_flush(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['a'] = tm.makeTimeSeries() store.flush() store.flush(fsync=True) def test_get(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['a'] = tm.makeTimeSeries() left = store.get('a') right = store['a'] @@ -495,7 +512,7 @@ def test_get(self): def test_getattr(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: s = tm.makeTimeSeries() store['a'] = s @@ -523,7 +540,7 @@ def test_getattr(self): def test_put(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: ts = tm.makeTimeSeries() df = tm.makeTimeDataFrame() @@ -552,7 +569,7 @@ def test_put(self): def test_put_string_index(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: index = Index( ["I am a very long string index: %s" % i for i in range(20)]) @@ -577,7 +594,7 @@ def test_put_string_index(self): def test_put_compression(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeTimeDataFrame() store.put('c', df, format='table', complib='zlib') @@ -591,7 +608,7 @@ def test_put_compression_blosc(self): tm.skip_if_no_package('tables', '2.2', app='blosc support') df = tm.makeTimeDataFrame() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # can't compress if format='fixed' self.assertRaises(ValueError, store.put, 'b', df, @@ -621,7 +638,7 @@ def test_put_mixed_type(self): df.ix[3:6, ['obj1']] = np.nan df = df.consolidate().convert_objects() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: _maybe_remove(store, 'df') # cannot use assert_produces_warning here for some reason @@ -635,7 +652,7 @@ def test_put_mixed_type(self): def test_append(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeTimeDataFrame() _maybe_remove(store, 'df1') store.append('df1', df[:10]) @@ -723,7 +740,7 @@ def test_append(self): def test_append_series(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # basic ss = tm.makeStringSeries() @@ -771,7 +788,7 @@ def test_store_index_types(self): # GH5386 # test storing various index types - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: def check(format,index): df = DataFrame(np.random.randn(10,2),columns=list('AB')) @@ -804,7 +821,7 @@ def test_encoding(self): if sys.byteorder != 'little': raise nose.SkipTest('system byteorder is not little, skipping test_encoding!') - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame(dict(A='foo',B='bar'),index=range(5)) df.loc[2,'A'] = np.nan df.loc[3,'B'] = np.nan @@ -818,7 +835,7 @@ def test_encoding(self): def test_append_some_nans(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame({'A' : Series(np.random.randn(20)).astype('int32'), 'A1' : np.random.randn(20), 'A2' : np.random.randn(20), @@ -857,7 +874,7 @@ def test_append_some_nans(self): def test_append_all_nans(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame({'A1' : np.random.randn(20), 'A2' : np.random.randn(20)}, @@ -928,7 +945,7 @@ def test_append_all_nans(self): def test_append_frame_column_oriented(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # column oriented df = tm.makeTimeDataFrame() @@ -954,7 +971,7 @@ def test_append_frame_column_oriented(self): def test_append_with_different_block_ordering(self): #GH 4096; using same frames, but different block orderings - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: for i in range(10): @@ -976,7 +993,7 @@ def test_append_with_different_block_ordering(self): store.append('df',df) # test a different ordering but with more fields (like invalid combinate) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame(np.random.randn(10,2),columns=list('AB'), dtype='float64') df['int64'] = Series([1]*len(df),dtype='int64') @@ -994,7 +1011,7 @@ def test_append_with_different_block_ordering(self): def test_ndim_indexables(self): """ test using ndim tables in new ways""" - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: p4d = tm.makePanel4D() @@ -1061,7 +1078,7 @@ def check_indexers(key, indexers): def test_append_with_strings(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: wp = tm.makePanel() wp2 = wp.rename_axis( dict([(x, "%s_extra" % x) for x in wp.minor_axis]), axis=2) @@ -1130,7 +1147,7 @@ def check_col(key,name,size): result = store.select('df') tm.assert_frame_equal(result, df) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: def check_col(key,name,size): self.assert_(getattr(store.get_storer(key).table.description,name).itemsize == size) @@ -1169,7 +1186,7 @@ def check_col(key,name,size): def test_append_with_data_columns(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeTimeDataFrame() df.loc[:,'B'].iloc[0] = 1. _maybe_remove(store, 'df') @@ -1208,7 +1225,7 @@ def test_append_with_data_columns(self): def check_col(key,name,size): self.assert_(getattr(store.get_storer(key).table.description,name).itemsize == size) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: _maybe_remove(store, 'df') store.append('df', df_new, data_columns=['string'], min_itemsize={'string': 30}) @@ -1222,7 +1239,7 @@ def check_col(key,name,size): min_itemsize={'values': 30}) check_col('df', 'string', 30) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df_new['string2'] = 'foobarbah' df_new['string_block1'] = 'foobarbah1' df_new['string_block2'] = 'foobarbah2' @@ -1232,7 +1249,7 @@ def check_col(key,name,size): check_col('df', 'string2', 40) check_col('df', 'values_block_1', 50) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # multiple data columns df_new = df.copy() df_new.loc[:,'A'].iloc[0] = 1. @@ -1259,7 +1276,7 @@ def check_col(key,name,size): df_new.string2 == 'cool')] tm.assert_frame_equal(result, expected) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # doc example df_dc = df.copy() df_dc['string'] = 'foo' @@ -1286,7 +1303,7 @@ def check_col(key,name,size): def test_create_table_index(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: def col(t,column): return getattr(store.get_storer(t).table.cols,column) @@ -1377,7 +1394,7 @@ def test_big_table_frame(self): import time x = time.time() - with ensure_clean(self.path,mode='w') as store: + with ensure_clean_store(self.path,mode='w') as store: store.append('df', df) rows = store.root.df.table.nrows recons = store.select('df') @@ -1405,7 +1422,7 @@ def test_big_table2_frame(self): % (len(df.index), time.time() - start_time)) def f(chunksize): - with ensure_clean(self.path,mode='w') as store: + with ensure_clean_store(self.path,mode='w') as store: store.append('df', df, chunksize=chunksize) r = store.root.df.table.nrows return r @@ -1433,7 +1450,7 @@ def test_big_put_frame(self): print("\nbig_put frame (creation of df) [rows->%s] -> %5.2f" % (len(df.index), time.time() - start_time)) - with ensure_clean(self.path, mode='w') as store: + with ensure_clean_store(self.path, mode='w') as store: start_time = time.time() store = HDFStore(self.path, mode='w') store.put('df', df) @@ -1459,7 +1476,7 @@ def test_big_table_panel(self): x = time.time() - with ensure_clean(self.path, mode='w') as store: + with ensure_clean_store(self.path, mode='w') as store: store.append('wp', wp) rows = store.root.wp.table.nrows recons = store.select('wp') @@ -1473,7 +1490,7 @@ def test_append_diff_item_order(self): wp1 = wp.ix[:, :10, :] wp2 = wp.ix[['ItemC', 'ItemB', 'ItemA'], 10:, :] - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('panel', wp1, format='table') self.assertRaises(ValueError, store.put, 'panel', wp2, append=True) @@ -1487,7 +1504,7 @@ def test_append_hierarchical(self): df = DataFrame(np.random.randn(10, 3), index=index, columns=['A', 'B', 'C']) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('mi', df) result = store.select('mi') tm.assert_frame_equal(result, df) @@ -1497,7 +1514,7 @@ def test_append_hierarchical(self): expected = df.reindex(columns=['A','B']) tm.assert_frame_equal(result,expected) - with tm.ensure_clean('test.hdf') as path: + with ensure_clean_path('test.hdf') as path: df.to_hdf(path,'df',format='table') result = read_hdf(path,'df',columns=['A','B']) expected = df.reindex(columns=['A','B']) @@ -1510,7 +1527,7 @@ def test_column_multiindex(self): index = MultiIndex.from_tuples([('A','a'), ('A','b'), ('B','a'), ('B','b')], names=['first','second']) df = DataFrame(np.arange(12).reshape(3,4), columns=index) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('df',df) tm.assert_frame_equal(store['df'],df,check_index_type=True,check_column_type=True) @@ -1524,7 +1541,7 @@ def test_column_multiindex(self): # non_index_axes name df = DataFrame(np.arange(12).reshape(3,4), columns=Index(list('ABCD'),name='foo')) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('df1',df,format='table') tm.assert_frame_equal(store['df1'],df,check_index_type=True,check_column_type=True) @@ -1533,14 +1550,14 @@ def test_pass_spec_to_storer(self): df = tm.makeDataFrame() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('df',df) self.assertRaises(TypeError, store.select, 'df', columns=['A']) self.assertRaises(TypeError, store.select, 'df',where=[('columns=A')]) def test_append_misc(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # unsuported data types for non-tables p4d = tm.makePanel4D() @@ -1564,7 +1581,7 @@ def test_append_misc(self): # more chunksize in append tests def check(obj, comparator): for c in [10, 200, 1000]: - with ensure_clean(self.path,mode='w') as store: + with ensure_clean_store(self.path,mode='w') as store: store.append('obj', obj, chunksize=c) result = store.select('obj') comparator(result,obj) @@ -1585,7 +1602,7 @@ def check(obj, comparator): check(p4d, assert_panel4d_equal) # empty frame, GH4273 - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # 0 len df_empty = DataFrame(columns=list('ABC')) @@ -1622,7 +1639,7 @@ def check(obj, comparator): def test_append_raise(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # test append with invalid input to get good error messages @@ -1664,14 +1681,14 @@ def test_table_index_incompatible_dtypes(self): df2 = DataFrame({'a': [4, 5, 6]}, index=date_range('1/1/2000', periods=3)) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('frame', df1, format='table') self.assertRaises(TypeError, store.put, 'frame', df2, format='table', append=True) def test_table_values_dtypes_roundtrip(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df1 = DataFrame({'a': [1, 2, 3]}, dtype='f8') store.append('df_f8', df1) assert_series_equal(df1.dtypes,store['df_f8'].dtypes) @@ -1726,7 +1743,7 @@ def test_table_mixed_dtypes(self): df.ix[3:6, ['obj1']] = np.nan df = df.consolidate().convert_objects() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('df1_mixed', df) tm.assert_frame_equal(store.select('df1_mixed'), df) @@ -1740,7 +1757,7 @@ def test_table_mixed_dtypes(self): wp['int2'] = 2 wp = wp.consolidate() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('p1_mixed', wp) assert_panel_equal(store.select('p1_mixed'), wp) @@ -1754,13 +1771,13 @@ def test_table_mixed_dtypes(self): wp['int2'] = 2 wp = wp.consolidate() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('p4d_mixed', wp) assert_panel4d_equal(store.select('p4d_mixed'), wp) def test_unimplemented_dtypes_table_columns(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: l = [('date', datetime.date(2001, 1, 2))] @@ -1782,7 +1799,7 @@ def test_unimplemented_dtypes_table_columns(self): df['datetime1'] = datetime.date(2001, 1, 2) df = df.consolidate().convert_objects() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # this fails because we have a date in the object block...... self.assertRaises(TypeError, store.append, 'df_unimplemented', df) @@ -1802,7 +1819,7 @@ def compare(a,b): raise AssertionError("invalid tz comparsion [%s] [%s]" % (a_e,b_e)) # as columns - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: _maybe_remove(store, 'df_tz') df = DataFrame(dict(A = [ Timestamp('20130102 2:00:00',tz='US/Eastern') + timedelta(hours=1)*i for i in range(5) ])) @@ -1837,7 +1854,7 @@ def compare(a,b): self.assertRaises(ValueError, store.append, 'df_tz', df) # as index - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # GH 4098 example df = DataFrame(dict(A = Series(lrange(3), index=date_range('2000-1-1',periods=3,freq='H', tz='US/Eastern')))) @@ -1865,7 +1882,7 @@ def test_store_timezone(self): import os # original method - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: today = datetime.date(2013,9,10) df = DataFrame([1,2,3], index = [today, today, today]) @@ -1888,7 +1905,7 @@ def setTZ(tz): try: - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: setTZ('EST5EDT') today = datetime.date(2013,9,10) @@ -1915,7 +1932,7 @@ def test_append_with_timedelta(self): df['C'] = df['A']-df['B'] df.ix[3:5,'C'] = np.nan - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # table _maybe_remove(store, 'df') @@ -1950,7 +1967,7 @@ def test_append_with_timedelta(self): def test_remove(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: ts = tm.makeTimeSeries() df = tm.makeDataFrame() @@ -1987,7 +2004,7 @@ def test_remove(self): def test_remove_where(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # non-existance crit1 = Term('index>foo') @@ -2023,7 +2040,7 @@ def test_remove_where(self): def test_remove_crit(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: wp = tm.makePanel() @@ -2089,7 +2106,7 @@ def test_remove_crit(self): def test_invalid_terms(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeTimeDataFrame() df['string'] = 'foo' @@ -2112,7 +2129,7 @@ def test_invalid_terms(self): self.assertRaises(ValueError, store.select, 'wp', "major_axis<'20000108' & minor_axis['A', 'B']") # from the docs - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: dfq = DataFrame(np.random.randn(10,4),columns=list('ABCD'),index=date_range('20130101',periods=10)) dfq.to_hdf(path,'dfq',format='table',data_columns=True) @@ -2121,7 +2138,7 @@ def test_invalid_terms(self): read_hdf(path,'dfq',where="A>0 or C>0") # catch the invalid reference - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: dfq = DataFrame(np.random.randn(10,4),columns=list('ABCD'),index=date_range('20130101',periods=10)) dfq.to_hdf(path,'dfq',format='table') @@ -2129,7 +2146,7 @@ def test_invalid_terms(self): def test_terms(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: wp = tm.makePanel() p4d = tm.makePanel4D() @@ -2196,7 +2213,7 @@ def test_terms(self): store.select('p4d', t) def test_term_compat(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: wp = Panel(np.random.randn(2, 5, 4), items=['Item1', 'Item2'], major_axis=date_range('1/1/2000', periods=5), @@ -2215,7 +2232,7 @@ def test_term_compat(self): def test_same_name_scoping(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: import pandas as pd df = DataFrame(np.random.randn(20, 2),index=pd.date_range('20130101',periods=20)) @@ -2390,7 +2407,7 @@ def test_frame(self): self._check_roundtrip(tdf, tm.assert_frame_equal, compression=True) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # not consolidated df['foo'] = np.random.randn(len(df)) store['df'] = df @@ -2429,7 +2446,7 @@ def test_timezones(self): rng = date_range('1/1/2000', '1/30/2000', tz='US/Eastern') frame = DataFrame(np.random.randn(len(rng), 4), index=rng) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['frame'] = frame recons = store['frame'] self.assert_(recons.index.equals(rng)) @@ -2439,7 +2456,7 @@ def test_fixed_offset_tz(self): rng = date_range('1/1/2000 00:00:00-07:00', '1/30/2000 00:00:00-07:00') frame = DataFrame(np.random.randn(len(rng), 4), index=rng) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['frame'] = frame recons = store['frame'] self.assert_(recons.index.equals(rng)) @@ -2459,7 +2476,7 @@ def test_store_hierarchical(self): self._check_roundtrip(frame['A'], tm.assert_series_equal) # check that the names are stored - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['frame'] = frame recons = store['frame'] assert(recons.index.names == ('foo', 'bar')) @@ -2468,7 +2485,7 @@ def test_store_index_name(self): df = tm.makeDataFrame() df.index.name = 'foo' - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['frame'] = df recons = store['frame'] assert(recons.index.name == 'foo') @@ -2477,7 +2494,7 @@ def test_store_series_name(self): df = tm.makeDataFrame() series = df['A'] - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['series'] = series recons = store['series'] assert(recons.name == 'A') @@ -2500,7 +2517,7 @@ def _make_one(): self._check_roundtrip(df1, tm.assert_frame_equal) self._check_roundtrip(df2, tm.assert_frame_equal) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['obj'] = df1 tm.assert_frame_equal(store['obj'], df1) store['obj'] = df2 @@ -2537,7 +2554,7 @@ def test_select_with_dups(self): df = DataFrame(np.random.randn(10,4),columns=['A','A','B','B']) df.index = date_range('20130101 9:30',periods=10,freq='T') - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('df',df) result = store.select('df') @@ -2558,7 +2575,7 @@ def test_select_with_dups(self): axis=1) df.index = date_range('20130101 9:30',periods=10,freq='T') - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('df',df) result = store.select('df') @@ -2578,7 +2595,7 @@ def test_select_with_dups(self): assert_frame_equal(result,expected,by_blocks=True) # duplicates on both index and columns - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.append('df',df) store.append('df',df) @@ -2589,7 +2606,7 @@ def test_select_with_dups(self): def test_wide_table_dups(self): wp = tm.makePanel() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('panel', wp, format='table') store.put('panel', wp, format='table', append=True) @@ -2613,7 +2630,7 @@ def test_longpanel(self): def test_overwrite_node(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store['a'] = tm.makeTimeDataFrame() ts = tm.makeTimeSeries() store['a'] = ts @@ -2653,7 +2670,7 @@ def test_sparse_with_compression(self): def test_select(self): wp = tm.makePanel() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # put/select ok _maybe_remove(store, 'wp') @@ -2717,7 +2734,7 @@ def test_select(self): def test_select_dtypes(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # with a Timestamp data column (GH #2637) df = DataFrame(dict(ts=bdate_range('2012-01-01', periods=300), A=np.random.randn(300))) @@ -2765,7 +2782,7 @@ def test_select_dtypes(self): expected = df.reindex(index=list(df.index)[0:10],columns=['A']) tm.assert_frame_equal(expected, result) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # floats w/o NaN df = DataFrame(dict(cols = range(11), values = range(11)),dtype='float64') @@ -2807,7 +2824,7 @@ def test_select_dtypes(self): def test_select_with_many_inputs(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame(dict(ts=bdate_range('2012-01-01', periods=300), A=np.random.randn(300), @@ -2848,7 +2865,7 @@ def test_select_with_many_inputs(self): def test_select_iterator(self): # single table - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeTimeDataFrame(500) _maybe_remove(store, 'df') @@ -2874,14 +2891,14 @@ def test_select_iterator(self): result = concat(results) tm.assert_frame_equal(result, expected) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeTimeDataFrame(500) df.to_hdf(path,'df_non_table') self.assertRaises(TypeError, read_hdf, path,'df_non_table',chunksize=100) self.assertRaises(TypeError, read_hdf, path,'df_non_table',iterator=True) - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeTimeDataFrame(500) df.to_hdf(path,'df',format='table') @@ -2897,7 +2914,7 @@ def test_select_iterator(self): # multiple - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df1 = tm.makeTimeDataFrame(500) store.append('df1',df1,data_columns=True) @@ -2933,7 +2950,7 @@ def test_retain_index_attributes(self): df = DataFrame(dict(A = Series(lrange(3), index=date_range('2000-1-1',periods=3,freq='H')))) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: _maybe_remove(store,'data') store.put('data', df, format='table') @@ -2963,7 +2980,7 @@ def test_retain_index_attributes(self): def test_retain_index_attributes2(self): - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: with tm.assert_produces_warning(expected_warning=AttributeConflictWarning): @@ -2992,7 +3009,7 @@ def test_panel_select(self): wp = tm.makePanel() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('wp', wp, format='table') date = wp.major_axis[len(wp.major_axis) // 2] @@ -3012,7 +3029,7 @@ def test_frame_select(self): df = tm.makeTimeDataFrame() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('frame', df,format='table') date = df.index[len(df) // 2] @@ -3046,7 +3063,7 @@ def test_frame_select_complex(self): df['string'] = 'foo' df.loc[df.index[0:4],'string'] = 'bar' - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('df', df, table=True, data_columns=['string']) # empty @@ -3091,7 +3108,7 @@ def test_invalid_filtering(self): df = tm.makeTimeDataFrame() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('df', df, table=True) # not implemented @@ -3103,7 +3120,7 @@ def test_invalid_filtering(self): def test_string_select(self): # GH 2973 - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = tm.makeTimeDataFrame() @@ -3152,7 +3169,7 @@ def test_read_column(self): df = tm.makeTimeDataFrame() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: _maybe_remove(store, 'df') store.append('df', df) @@ -3190,7 +3207,7 @@ def f(): def test_coordinates(self): df = tm.makeTimeDataFrame() - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: _maybe_remove(store, 'df') store.append('df', df) @@ -3235,7 +3252,7 @@ def test_coordinates(self): tm.assert_frame_equal(result, expected) # pass array/mask as the coordinates - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame(np.random.randn(1000,2),index=date_range('20000101',periods=1000)) store.append('df',df) @@ -3277,7 +3294,7 @@ def test_append_to_multiple(self): df2['foo'] = 'bar' df = concat([df1, df2], axis=1) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # exceptions self.assertRaises(ValueError, store.append_to_multiple, @@ -3301,7 +3318,7 @@ def test_append_to_multiple_dropna(self): df1.ix[1, ['A', 'B']] = np.nan df = concat([df1, df2], axis=1) - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # dropna=True should guarantee rows are synchronized store.append_to_multiple( {'df1': ['A', 'B'], 'df2': None}, df, selector='df1', @@ -3327,7 +3344,7 @@ def test_select_as_multiple(self): df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2" % x) df2['foo'] = 'bar' - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: # no tables stored self.assertRaises(Exception, store.select_as_multiple, @@ -3378,7 +3395,7 @@ def test_select_as_multiple(self): def test_start_stop(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: df = DataFrame(dict(A=np.random.rand(20), B=np.random.rand(20))) store.append('df', df) @@ -3400,7 +3417,7 @@ def test_select_filter_corner(self): df.index = ['%.3d' % c for c in df.index] df.columns = ['%.3d' % c for c in df.columns] - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: store.put('frame', df, format='table') crit = Term('columns=df.columns[:75]') @@ -3413,7 +3430,7 @@ def _check_roundtrip(self, obj, comparator, compression=False, **kwargs): if compression: options['complib'] = _default_compressor - with ensure_clean(self.path, 'w', **options) as store: + with ensure_clean_store(self.path, 'w', **options) as store: store['obj'] = obj retrieved = store['obj'] comparator(retrieved, obj, **kwargs) @@ -3424,7 +3441,7 @@ def _check_double_roundtrip(self, obj, comparator, compression=False, if compression: options['complib'] = compression or _default_compressor - with ensure_clean(self.path, 'w', **options) as store: + with ensure_clean_store(self.path, 'w', **options) as store: store['obj'] = obj retrieved = store['obj'] comparator(retrieved, obj, **kwargs) @@ -3437,7 +3454,7 @@ def _check_roundtrip_table(self, obj, comparator, compression=False): if compression: options['complib'] = _default_compressor - with ensure_clean(self.path, 'w', **options) as store: + with ensure_clean_store(self.path, 'w', **options) as store: store.put('obj', obj, format='table') retrieved = store['obj'] # sorted_obj = _test_sort(obj) @@ -3446,7 +3463,7 @@ def _check_roundtrip_table(self, obj, comparator, compression=False): def test_multiple_open_close(self): # GH 4409, open & close multiple times - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeDataFrame() df.to_hdf(path,'df',mode='w',format='table') @@ -3508,7 +3525,7 @@ def test_multiple_open_close(self): self.assert_(not store2.is_open) # ops on a closed store - with tm.ensure_clean(self.path) as path: + with ensure_clean_path(self.path) as path: df = tm.makeDataFrame() df.to_hdf(path,'df',mode='w',format='table') @@ -3693,7 +3710,7 @@ def test_legacy_table_write(self): def test_store_datetime_fractional_secs(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: dt = datetime.datetime(2012, 1, 2, 3, 4, 5, 123456) series = Series([0], [dt]) store['a'] = series @@ -3701,7 +3718,7 @@ def test_store_datetime_fractional_secs(self): def test_tseries_indices_series(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: idx = tm.makeDateIndex(10) ser = Series(np.random.randn(len(idx)), idx) store['a'] = ser @@ -3722,7 +3739,7 @@ def test_tseries_indices_series(self): def test_tseries_indices_frame(self): - with ensure_clean(self.path) as store: + with ensure_clean_store(self.path) as store: idx = tm.makeDateIndex(10) df = DataFrame(np.random.randn(len(idx), 3), index=idx) store['a'] = df @@ -3773,7 +3790,7 @@ def test_append_with_diff_col_name_types_raises_value_error(self): df4 = DataFrame({('1', 2): np.random.randn(10)}) df5 = DataFrame({('1', 2, object): np.random.randn(10)}) - with ensure_clean('__%s__.h5' % tm.rands(20)) as store: + with ensure_clean_store(self.path) as store: name = 'df_%s' % tm.rands(10) store.append(name, df) diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 9e742f2065419..895c651c0bfe7 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -334,7 +334,7 @@ def ensure_clean(filename=None, return_filelike=False): raise ValueError("Can't pass a qualified name to ensure_clean()") try: - filename = create_temp_file(filename) + filename = tempfile.mkstemp(suffix=filename)[1] yield filename finally: try: @@ -343,13 +343,6 @@ def ensure_clean(filename=None, return_filelike=False): except Exception as e: print("Exception on removing file: %s" % e) - -def create_temp_file(filename): - """ - create a temporary file. the caller is responsible for deleting the file - """ - return tempfile.mktemp(suffix=filename) - def get_data_path(f=''): """Return the path of a data file, these are relative to the current test directory.