From 2cda89df0871531531cf50529c53a58ac63e05ad Mon Sep 17 00:00:00 2001 From: hugo Date: Sun, 9 Feb 2014 18:31:50 -0500 Subject: [PATCH 1/4] FIX: hdfstore queries of the form where=[('date', '>=', datetime(2013,1,1)), ('date', '<=', datetime(2014,1,1))] were broken - modified Expr.parse_back_compat to check for tuples, in w, and unpack into w, op, value - modified Expr.__init__ to modify the where list/tuple with the parsed result --- pandas/computation/pytables.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pandas/computation/pytables.py b/pandas/computation/pytables.py index bf477cd71df62..c5b0785fe6f72 100644 --- a/pandas/computation/pytables.py +++ b/pandas/computation/pytables.py @@ -488,7 +488,6 @@ def __init__(self, where, op=None, value=None, queryables=None, self.filter = None self.terms = None self._visitor = None - # capture the environement if needed lcls = dict() if isinstance(where, Expr): @@ -497,13 +496,12 @@ def __init__(self, where, op=None, value=None, queryables=None, where = where.expr elif isinstance(where, (list, tuple)): - - for w in where: + for idx, w in enumerate(where): if isinstance(w, Expr): lcls.update(w.env.locals) else: w = self.parse_back_compat(w) - + where[idx] = w where = ' & ' .join(["(%s)" % w for w in where]) self.expr = where @@ -528,7 +526,16 @@ def parse_back_compat(self, w, op=None, value=None): warnings.warn("passing a dict to Expr is deprecated, " "pass the where as a single string", DeprecationWarning) - + if isinstance(w, tuple): + if len(w) == 2: + w, value = w + op = '==' + elif len(w) == 3: + w, op, value = w + warnings.warn("passing a tuple into Expr is deprecated, " + "pass the where as a single string", + DeprecationWarning) + if op is not None: if not isinstance(w, string_types): raise TypeError( From 46283e0c4f282e38358f8b83f464e8bf726c25e7 Mon Sep 17 00:00:00 2001 From: hugo Date: Sun, 9 Feb 2014 19:09:55 -0500 Subject: [PATCH 2/4] added test by taking test_term_compat, and removing all Term calls --- pandas/io/tests/test_pytables.py | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py index b12915753127d..45259e20dbd41 100644 --- a/pandas/io/tests/test_pytables.py +++ b/pandas/io/tests/test_pytables.py @@ -2474,6 +2474,48 @@ def test_term_compat(self): expected = wp.loc[:,:,['A','B']] assert_panel_equal(result, expected) + def test_backwards_compat_without_term_object(self): + with ensure_clean_store(self.path) as store: + + wp = Panel(np.random.randn(2, 5, 4), items=['Item1', 'Item2'], + major_axis=date_range('1/1/2000', periods=5), + minor_axis=['A', 'B', 'C', 'D']) + store.append('wp',wp) + + result = store.select('wp', [('major_axis>20000102'), + ('minor_axis', '=', ['A','B']) ]) + expected = wp.loc[:,wp.major_axis>Timestamp('20000102'),['A','B']] + assert_panel_equal(result, expected) + + store.remove('wp', ('major_axis>20000103')) + result = store.select('wp') + expected = wp.loc[:,wp.major_axis<=Timestamp('20000103'),:] + assert_panel_equal(result, expected) + + with ensure_clean_store(self.path) as store: + + wp = Panel(np.random.randn(2, 5, 4), items=['Item1', 'Item2'], + major_axis=date_range('1/1/2000', periods=5), + minor_axis=['A', 'B', 'C', 'D']) + store.append('wp',wp) + + # stringified datetimes + result = store.select('wp', [('major_axis','>',datetime.datetime(2000,1,2))]) + expected = wp.loc[:,wp.major_axis>Timestamp('20000102')] + assert_panel_equal(result, expected) + + result = store.select('wp', [('major_axis','>',datetime.datetime(2000,1,2,0,0))]) + expected = wp.loc[:,wp.major_axis>Timestamp('20000102')] + assert_panel_equal(result, expected) + + result = store.select('wp', [('major_axis','=',[datetime.datetime(2000,1,2,0,0),datetime.datetime(2000,1,3,0,0)])]) + expected = wp.loc[:,[Timestamp('20000102'),Timestamp('20000103')]] + assert_panel_equal(result, expected) + + result = store.select('wp', [('minor_axis','=',['A','B'])]) + expected = wp.loc[:,:,['A','B']] + assert_panel_equal(result, expected) + def test_same_name_scoping(self): with ensure_clean_store(self.path) as store: From a608e145d662d9999f49fad379d1016e08cda63a Mon Sep 17 00:00:00 2001 From: hugo Date: Thu, 13 Feb 2014 23:20:25 -0500 Subject: [PATCH 3/4] DOC: added release note about #6313 --- doc/source/release.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/release.rst b/doc/source/release.rst index 5aeea685b8ff4..be6e213f12183 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -124,6 +124,7 @@ Bug Fixes keys not in the values to be replaced (:issue:`6342`) - Bug in take with duplicate columns not consolidated (:issue:`6240`) - Bug in interpolate changing dtypes (:issue:`6290`) +- Bug in hdfstore queries of the form ``where=[('date', '>=', datetime(2013,1,1)), ('date', '<=', datetime(2014,1,1))]`` (:issue:`6313`) pandas 0.13.1 ------------- From 3ca1afeca464954c94492d93c76c00839418b901 Mon Sep 17 00:00:00 2001 From: hugo Date: Mon, 17 Feb 2014 14:40:56 -0500 Subject: [PATCH 4/4] TST: checked for DeprecationWarning on tests for backwards compatability logic --- pandas/io/tests/test_pytables.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py index 45259e20dbd41..5380260cb5a44 100644 --- a/pandas/io/tests/test_pytables.py +++ b/pandas/io/tests/test_pytables.py @@ -2481,9 +2481,9 @@ def test_backwards_compat_without_term_object(self): major_axis=date_range('1/1/2000', periods=5), minor_axis=['A', 'B', 'C', 'D']) store.append('wp',wp) - - result = store.select('wp', [('major_axis>20000102'), - ('minor_axis', '=', ['A','B']) ]) + with tm.assert_produces_warning(expected_warning=DeprecationWarning): + result = store.select('wp', [('major_axis>20000102'), + ('minor_axis', '=', ['A','B']) ]) expected = wp.loc[:,wp.major_axis>Timestamp('20000102'),['A','B']] assert_panel_equal(result, expected) @@ -2500,22 +2500,24 @@ def test_backwards_compat_without_term_object(self): store.append('wp',wp) # stringified datetimes - result = store.select('wp', [('major_axis','>',datetime.datetime(2000,1,2))]) + with tm.assert_produces_warning(expected_warning=DeprecationWarning): + result = store.select('wp', [('major_axis','>',datetime.datetime(2000,1,2))]) expected = wp.loc[:,wp.major_axis>Timestamp('20000102')] assert_panel_equal(result, expected) - - result = store.select('wp', [('major_axis','>',datetime.datetime(2000,1,2,0,0))]) + with tm.assert_produces_warning(expected_warning=DeprecationWarning): + result = store.select('wp', [('major_axis','>',datetime.datetime(2000,1,2,0,0))]) expected = wp.loc[:,wp.major_axis>Timestamp('20000102')] assert_panel_equal(result, expected) - - result = store.select('wp', [('major_axis','=',[datetime.datetime(2000,1,2,0,0),datetime.datetime(2000,1,3,0,0)])]) + with tm.assert_produces_warning(expected_warning=DeprecationWarning): + result = store.select('wp', [('major_axis','=',[datetime.datetime(2000,1,2,0,0), + datetime.datetime(2000,1,3,0,0)])]) expected = wp.loc[:,[Timestamp('20000102'),Timestamp('20000103')]] assert_panel_equal(result, expected) - - result = store.select('wp', [('minor_axis','=',['A','B'])]) + with tm.assert_produces_warning(expected_warning=DeprecationWarning): + result = store.select('wp', [('minor_axis','=',['A','B'])]) expected = wp.loc[:,:,['A','B']] assert_panel_equal(result, expected) - + def test_same_name_scoping(self): with ensure_clean_store(self.path) as store: