Skip to content

Commit e6b4e60

Browse files
committed
API: add io.hdf.default_format option to enable an override to default storage formats in HDFStore
1 parent 3a52e0a commit e6b4e60

File tree

3 files changed

+62
-4
lines changed

3 files changed

+62
-4
lines changed

doc/source/io.rst

+5-1
Original file line numberDiff line numberDiff line change
@@ -1831,7 +1831,11 @@ Table Format
18311831
format. Conceptually a ``table`` is shaped very much like a DataFrame,
18321832
with rows and columns. A ``table`` may be appended to in the same or
18331833
other sessions. In addition, delete & query type operations are
1834-
supported. This format is specified by ``format='table'`` or ``format='t'`` to ``append`` or ``put`` or ``to_hdf``
1834+
supported. This format is specified by ``format='table'`` or ``format='t'``
1835+
to ``append`` or ``put`` or ``to_hdf``
1836+
1837+
This format can be set as an option as well ``pd.set_option('io.hdf.default_format','table')`` to
1838+
enable ``put/append/to_hdf`` to by default store in the ``table`` format.
18351839
18361840
.. ipython:: python
18371841
:suppress:

pandas/io/pytables.py

+14-3
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,17 @@ class DuplicateWarning(Warning):
172172
: boolean
173173
drop ALL nan rows when appending to a table
174174
"""
175+
format_doc = """
176+
: format
177+
default format writing format, if None, then
178+
put will default to 'fixed' and append will default to 'table'
179+
"""
175180

176181
with config.config_prefix('io.hdf'):
177182
config.register_option('dropna_table', True, dropna_doc,
178183
validator=config.is_bool)
184+
config.register_option('default_format', None, format_doc,
185+
validator=config.is_one_of_factory(['fixed','table',None]))
179186

180187
# oh the troubles to reduce import time
181188
_table_mod = None
@@ -228,7 +235,7 @@ def get_store(path, **kwargs):
228235

229236
# interface to/from ###
230237

231-
def to_hdf(path_or_buf, key, value, mode=None, format=None, complevel=None, complib=None, append=None, **kwargs):
238+
def to_hdf(path_or_buf, key, value, mode=None, complevel=None, complib=None, append=None, **kwargs):
232239
""" store this object, close it if we opened it """
233240
if append:
234241
f = lambda store: store.append(key, value, **kwargs)
@@ -685,7 +692,9 @@ def put(self, key, value, format=None, append=False, **kwargs):
685692
For Table format, append the input data to the existing
686693
encoding : default None, provide an encoding for strings
687694
"""
688-
kwargs = self._validate_format(format or 'fixed', kwargs)
695+
if format is None:
696+
format = get_option("io.hdf.default_format") or 'fixed'
697+
kwargs = self._validate_format(format, kwargs)
689698
self._write_to_group(key, value, append=append, **kwargs)
690699

691700
def remove(self, key, where=None, start=None, stop=None):
@@ -771,7 +780,9 @@ def append(self, key, value, format=None, append=True, columns=None, dropna=None
771780

772781
if dropna is None:
773782
dropna = get_option("io.hdf.dropna_table")
774-
kwargs = self._validate_format(format or 'table', kwargs)
783+
if format is None:
784+
format = get_option("io.hdf.default_format") or 'table'
785+
kwargs = self._validate_format(format, kwargs)
775786
self._write_to_group(key, value, append=append, dropna=dropna, **kwargs)
776787

777788
def append_to_multiple(self, d, value, selector, data_columns=None, axes=None, **kwargs):

pandas/io/tests/test_pytables.py

+43
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,49 @@ def test_api(self):
222222
self.assertRaises(TypeError, df.to_hdf, path,'df',append=True,format='foo')
223223
self.assertRaises(TypeError, df.to_hdf, path,'df',append=False,format='bar')
224224

225+
226+
def test_api_default_format(self):
227+
228+
# default_format option
229+
with ensure_clean(self.path) as store:
230+
df = tm.makeDataFrame()
231+
232+
pandas.set_option('io.hdf.default_format','fixed')
233+
_maybe_remove(store,'df')
234+
store.put('df',df)
235+
self.assert_(not store.get_storer('df').is_table)
236+
self.assertRaises(ValueError, store.append, 'df2',df)
237+
238+
pandas.set_option('io.hdf.default_format','table')
239+
_maybe_remove(store,'df')
240+
store.put('df',df)
241+
self.assert_(store.get_storer('df').is_table)
242+
_maybe_remove(store,'df2')
243+
store.append('df2',df)
244+
self.assert_(store.get_storer('df').is_table)
245+
246+
pandas.set_option('io.hdf.default_format',None)
247+
248+
with tm.ensure_clean(self.path) as path:
249+
250+
df = tm.makeDataFrame()
251+
252+
pandas.set_option('io.hdf.default_format','fixed')
253+
df.to_hdf(path,'df')
254+
with get_store(path) as store:
255+
self.assert_(not store.get_storer('df').is_table)
256+
self.assertRaises(ValueError, df.to_hdf, path,'df2', append=True)
257+
258+
pandas.set_option('io.hdf.default_format','table')
259+
df.to_hdf(path,'df3')
260+
with get_store(path) as store:
261+
self.assert_(store.get_storer('df3').is_table)
262+
df.to_hdf(path,'df4',append=True)
263+
with get_store(path) as store:
264+
self.assert_(store.get_storer('df4').is_table)
265+
266+
pandas.set_option('io.hdf.default_format',None)
267+
225268
def test_keys(self):
226269

227270
with ensure_clean(self.path) as store:

0 commit comments

Comments
 (0)