Skip to content

COMPAT: provide compat tests and better error messages for multiple file opening on PyTables >= 3.1 (GH6047) #6104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 26, 2014
Merged

COMPAT: provide compat tests and better error messages for multiple file opening on PyTables >= 3.1 (GH6047) #6104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions pandas/io/pytables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,11 +211,12 @@ class DuplicateWarning(Warning):
# oh the troubles to reduce import time
_table_mod = None
_table_supports_index = False

_table_file_open_policy_is_strict = False

def _tables():
global _table_mod
global _table_supports_index
global _table_file_open_policy_is_strict
if _table_mod is None:
import tables
from distutils.version import LooseVersion
Expand All @@ -225,8 +226,15 @@ def _tables():
ver = tables.__version__
_table_supports_index = LooseVersion(ver) >= '2.3'

return _table_mod
# set the file open policy
# return the file open policy; this changes as of pytables 3.1
# depending on the HDF5 version
try:
_table_file_open_policy_is_strict = tables.file._FILE_OPEN_POLICY == 'strict'
except:
pass

return _table_mod

@contextmanager
def get_store(path, **kwargs):
Expand Down Expand Up @@ -524,6 +532,22 @@ def open(self, mode='a', **kwargs):
self._handle = tables.openFile(self._path, 'r', **kwargs)
else:
raise

except (ValueError) as e:

# trap PyTables >= 3.1 FILE_OPEN_POLICY exception
# to provide an updated message
if 'FILE_OPEN_POLICY' in str(e):

e = ValueError("PyTables [{version}] no longer supports opening multiple files\n"
"even in read-only mode on this HDF5 version [{hdf_version}]. You can accept this\n"
"and not open the same file multiple times at once,\n"
"upgrade the HDF5 version, or downgrade to PyTables 3.0.0 which allows\n"
"files to be opened multiple times at once\n".format(version=tables.__version__,
hdf_version=tables.getHDF5Version()))

raise e

except (Exception) as e:

# trying to read from a non-existant file causes an error which
Expand Down
98 changes: 56 additions & 42 deletions pandas/io/tests/test_pytables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
IncompatibilityWarning, PerformanceWarning,
AttributeConflictWarning, DuplicateWarning,
PossibleDataLossError, ClosedFileError)
from pandas.io import pytables as pytables
import pandas.util.testing as tm
from pandas.util.testing import (assert_panel4d_equal,
assert_panel_equal,
Expand Down Expand Up @@ -3683,53 +3684,66 @@ def test_multiple_open_close(self):
self.assert_('CLOSED' in str(store))
self.assert_(not store.is_open)

# multiples
store1 = HDFStore(path)
store2 = HDFStore(path)

self.assert_('CLOSED' not in str(store1))
self.assert_('CLOSED' not in str(store2))
self.assert_(store1.is_open)
self.assert_(store2.is_open)

store1.close()
self.assert_('CLOSED' in str(store1))
self.assert_(not store1.is_open)
self.assert_('CLOSED' not in str(store2))
self.assert_(store2.is_open)

store2.close()
self.assert_('CLOSED' in str(store1))
self.assert_('CLOSED' in str(store2))
self.assert_(not store1.is_open)
self.assert_(not store2.is_open)

# nested close
store = HDFStore(path,mode='w')
store.append('df',df)
with ensure_clean_path(self.path) as path:

store2 = HDFStore(path)
store2.append('df2',df)
store2.close()
self.assert_('CLOSED' in str(store2))
self.assert_(not store2.is_open)
if pytables._table_file_open_policy_is_strict:

store.close()
self.assert_('CLOSED' in str(store))
self.assert_(not store.is_open)
# multiples
store1 = HDFStore(path)
def f():
HDFStore(path)
self.assertRaises(ValueError, f)
store1.close()

# double closing
store = HDFStore(path,mode='w')
store.append('df', df)
else:

store2 = HDFStore(path)
store.close()
self.assert_('CLOSED' in str(store))
self.assert_(not store.is_open)
# multiples
store1 = HDFStore(path)
store2 = HDFStore(path)

self.assert_('CLOSED' not in str(store1))
self.assert_('CLOSED' not in str(store2))
self.assert_(store1.is_open)
self.assert_(store2.is_open)

store1.close()
self.assert_('CLOSED' in str(store1))
self.assert_(not store1.is_open)
self.assert_('CLOSED' not in str(store2))
self.assert_(store2.is_open)

store2.close()
self.assert_('CLOSED' in str(store1))
self.assert_('CLOSED' in str(store2))
self.assert_(not store1.is_open)
self.assert_(not store2.is_open)

# nested close
store = HDFStore(path,mode='w')
store.append('df',df)

store2 = HDFStore(path)
store2.append('df2',df)
store2.close()
self.assert_('CLOSED' in str(store2))
self.assert_(not store2.is_open)

store.close()
self.assert_('CLOSED' in str(store))
self.assert_(not store.is_open)

# double closing
store = HDFStore(path,mode='w')
store.append('df', df)

store2 = HDFStore(path)
store.close()
self.assert_('CLOSED' in str(store))
self.assert_(not store.is_open)

store2.close()
self.assert_('CLOSED' in str(store2))
self.assert_(not store2.is_open)
store2.close()
self.assert_('CLOSED' in str(store2))
self.assert_(not store2.is_open)

# ops on a closed store
with ensure_clean_path(self.path) as path:
Expand Down