BUG: syntax error in hdf query with ts

closes pandas-dev#15492

Author: Chris <[email protected]>

Closes pandas-dev#15544 from chris-b1/hdf-dt-error and squashes the following commits:

8288dca [Chris] lint
7c7100d [Chris] expand test cases
946a48e [Chris] ERR: more strict HDFStore string comparison
213585f [Chris] CLN: remove timetuple type check
cc977f0 [Chris] BUG: syntax error in hdf query with ts

Author: chris-b1

doc/source/whatsnew/v0.20.0.txt

@@ -501,6 +501,35 @@ New Behavior:
 
    df.groupby('A').agg([np.mean, np.std, np.min, np.max])
 
+.. _whatsnew_0200.api_breaking.hdfstore_where:
+
+HDFStore where string comparison
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In previous versions most types could be compared to string column in a ``HDFStore``
+usually resulting in an invalid comparsion.  These comparisions will now raise a
+``TypeError`` (:issue:`15492`)
+
+New Behavior:
+
+.. code-block:: ipython
+
+   In [15]: df = pd.DataFrame({'unparsed_date': ['2014-01-01', '2014-01-01']})
+
+   In [16]: df.dtypes
+   Out[16]:
+   unparsed_date    object
+   dtype: object
+
+   In [17]: df.to_hdf('store.h5', 'key', format='table', data_columns=True)
+
+   In [18]: ts = pd.Timestamp('2014-01-01')
+
+   In [19]: pd.read_hdf('store.h5', 'key', where='unparsed_date > ts')
+   TypeError: Cannot compare 2014-01-01 00:00:00 of
+   type <class 'pandas.tslib.Timestamp'> to string column
+
+
 .. _whatsnew_0200.api:
 
 Other API Changes
@@ -671,7 +700,7 @@ Bug Fixes
 - Bug in ``pd.merge_asof()`` where ``left_index``/``right_index`` together caused a failure when ``tolerance`` was specified (:issue:`15135`)
 
 
-
+- Bug in ``pd.read_hdf()`` passing a ``Timestamp`` to the ``where`` parameter with a non date column (:issue:`15492`)
 
 
 - Bug in ``Series`` constructor when both ``copy=True`` and ``dtype`` arguments are provided (:issue:`15125`)

pandas/computation/pytables.py

@@ -1,7 +1,6 @@
 """ manage PyTables query interface via Expressions """
 
 import ast
-import time
 import warnings
 from functools import partial
 from datetime import datetime, timedelta
@@ -188,10 +187,6 @@ def stringify(value):
             if v.tz is not None:
                 v = v.tz_convert('UTC')
             return TermValue(v, v.value, kind)
-        elif (isinstance(v, datetime) or hasattr(v, 'timetuple') or
-                kind == u('date')):
-            v = time.mktime(v.timetuple())
-            return TermValue(v, pd.Timestamp(v), kind)
         elif kind == u('timedelta64') or kind == u('timedelta'):
             v = _coerce_scalar_to_timedelta_type(v, unit='s').value
             return TermValue(int(v), v, kind)
@@ -218,12 +213,13 @@ def stringify(value):
             else:
                 v = bool(v)
             return TermValue(v, v, kind)
-        elif not isinstance(v, string_types):
-            v = stringify(v)
+        elif isinstance(v, string_types):
+            # string quoting
             return TermValue(v, stringify(v), u('string'))
-
-        # string quoting
-        return TermValue(v, stringify(v), u('string'))
+        else:
+            raise TypeError(("Cannot compare {v} of type {typ}"
+                            " to {kind} column").format(v=v, typ=type(v),
+                                                        kind=kind))
 
     def convert_values(self):
         pass
@@ -558,9 +554,8 @@ def parse_back_compat(self, w, op=None, value=None):
 
                 # stringify with quotes these values
                 def convert(v):
-                    if (isinstance(v, (datetime, np.datetime64,
-                                       timedelta, np.timedelta64)) or
-                            hasattr(v, 'timetuple')):
+                    if isinstance(v, (datetime, np.datetime64,
+                                      timedelta, np.timedelta64)):
                         return "'{0}'".format(v)
                     return v
 

pandas/tests/io/test_pytables.py

@@ -5071,6 +5071,50 @@ def test_query_long_float_literal(self):
             expected = df.loc[[1], :]
             tm.assert_frame_equal(expected, result)
 
+    def test_query_compare_column_type(self):
+        # GH 15492
+        df = pd.DataFrame({'date': ['2014-01-01', '2014-01-02'],
+                           'real_date': date_range('2014-01-01', periods=2),
+                           'float': [1.1, 1.2],
+                           'int': [1, 2]},
+                          columns=['date', 'real_date', 'float', 'int'])
+
+        with ensure_clean_store(self.path) as store:
+            store.append('test', df, format='table', data_columns=True)
+
+            ts = pd.Timestamp('2014-01-01') # noqa
+            result = store.select('test', where='real_date > ts')
+            expected = df.loc[[1], :]
+            tm.assert_frame_equal(expected, result)
+
+            for op in ['<', '>', '==']:
+                # non strings to string column always fail
+                for v in [2.1, True, pd.Timestamp('2014-01-01'),
+                          pd.Timedelta(1, 's')]:
+                    query = 'date {op} v'.format(op=op)
+                    with tm.assertRaises(TypeError):
+                        result = store.select('test', where=query)
+
+                # strings to other columns must be convertible to type
+                v = 'a'
+                for col in ['int', 'float', 'real_date']:
+                    query = '{col} {op} v'.format(op=op, col=col)
+                    with tm.assertRaises(ValueError):
+                        result = store.select('test', where=query)
+
+                for v, col in zip(['1', '1.1', '2014-01-01'],
+                                  ['int', 'float', 'real_date']):
+                    query = '{col} {op} v'.format(op=op, col=col)
+                    result = store.select('test', where=query)
+
+                    if op == '==':
+                        expected = df.loc[[0], :]
+                    elif op == '>':
+                        expected = df.loc[[1], :]
+                    else:
+                        expected = df.loc[[], :]
+                    tm.assert_frame_equal(expected, result)
+
 
 class TestHDFComplexValues(Base):
     # GH10447