[WiP] API: to_datetime, required unit with numerical (pandas-dev#15836)

* Tests
  * add test_to_datetime_numerical_input
  * add fixtures
  * update bool tests
* Check argument
  * for numerical type
  * DataFrame with unit

Author: atbd

pandas/_libs/tslib.pyx

@@ -2305,10 +2305,14 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
                             raise AssertionError
                         iresult[i] = NPY_NAT
 
+            elif util.is_bool_object(val):
+                raise TypeError("{0} is not convertible to datetime"
+                                .format(type(val)))
+
             else:
 
                 if is_raise:
-                    raise ValueError("non convertible value {0}"
+                    raise ValueError("non convertible value {0} "
                                      "with the unit '{1}'".format(
                                          val,
                                          unit))

pandas/core/indexes/datetimes.py

@@ -332,7 +332,7 @@ def __new__(cls, data=None,
         if not (is_datetime64_dtype(data) or is_datetimetz(data) or
                 is_integer_dtype(data)):
             data = tools.to_datetime(data, dayfirst=dayfirst,
-                                     yearfirst=yearfirst)
+                                     unit='ns', yearfirst=yearfirst)
 
         if issubclass(data.dtype.type, np.datetime64) or is_datetimetz(data):
 

pandas/core/tools/datetimes.py

@@ -14,7 +14,8 @@
     is_float,
     is_list_like,
     is_scalar,
-    is_numeric_dtype)
+    is_numeric_dtype,
+    is_bool_dtype)
 from pandas.core.dtypes.generic import (
     ABCIndexClass, ABCSeries,
     ABCDataFrame)
@@ -337,6 +338,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
 
     """
     from pandas.core.indexes.datetimes import DatetimeIndex
+    from pandas import DataFrame
 
     tz = 'utc' if utc else None
 
@@ -446,9 +448,24 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
             except (ValueError, TypeError):
                 raise e
 
+    def check_numerical_arg():
+        return ((is_scalar(arg) and (is_integer(arg) or is_float(arg))) or
+                is_numeric_dtype(np.asarray(arg)))
+
     if arg is None:
         return None
 
+    if unit is not None:
+        if isinstance(arg, (DataFrame,)):
+            raise ValueError("unit must be None if arg is a DataFrame")
+        if is_bool_dtype(np.asarray(arg)):
+            raise TypeError("{0} is not convertible to datetime"
+                            .format(type(arg)))
+    else:
+        if (format is None and check_numerical_arg() and
+            not isinstance(arg, (DataFrame,)) and np.asarray(arg).size):
+            raise ValueError("a unit is required in case of numerical arg")
+
     # handle origin
     if origin == 'julian':
 
@@ -474,8 +491,7 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
 
         # arg must be a numeric
         original = arg
-        if not ((is_scalar(arg) and (is_integer(arg) or is_float(arg))) or
-                is_numeric_dtype(np.asarray(arg))):
+        if not check_numerical_arg():
             raise ValueError(
                 "'{arg}' is not compatible with origin='{origin}'; "
                 "it must be numeric with a unit specified ".format(

pandas/tests/indexes/datetimes/test_tools.py

@@ -306,25 +306,6 @@ def test_to_datetime_tz_psycopg2(self):
                                     dtype='datetime64[ns, UTC]')
         tm.assert_index_equal(result, expected)
 
-    def test_datetime_bool(self):
-        # GH13176
-        with pytest.raises(TypeError):
-            to_datetime(False)
-        assert to_datetime(False, errors="coerce") is NaT
-        assert to_datetime(False, errors="ignore") is False
-        with pytest.raises(TypeError):
-            to_datetime(True)
-        assert to_datetime(True, errors="coerce") is NaT
-        assert to_datetime(True, errors="ignore") is True
-        with pytest.raises(TypeError):
-            to_datetime([False, datetime.today()])
-        with pytest.raises(TypeError):
-            to_datetime(['20130101', True])
-        tm.assert_index_equal(to_datetime([0, False, NaT, 0.0],
-                                          errors="coerce"),
-                              DatetimeIndex([to_datetime(0), NaT,
-                                             NaT, to_datetime(0)]))
-
     def test_datetime_invalid_datatype(self):
         # GH13176
 
@@ -409,10 +390,10 @@ def test_unit_with_numeric(self):
         arr1 = [1.434692e+18, 1.432766e+18]
         arr2 = np.array(arr1).astype('int64')
         for errors in ['ignore', 'raise', 'coerce']:
-            result = pd.to_datetime(arr1, errors=errors)
+            result = pd.to_datetime(arr1, unit='ns', errors=errors)
             tm.assert_index_equal(result, expected)
 
-            result = pd.to_datetime(arr2, errors=errors)
+            result = pd.to_datetime(arr2, unit='ns', errors=errors)
             tm.assert_index_equal(result, expected)
 
         # but we want to make sure that we are coercing
@@ -421,15 +402,15 @@ def test_unit_with_numeric(self):
                                   '2015-06-19 05:33:20',
                                   '2015-05-27 22:33:20'])
         arr = ['foo', 1.434692e+18, 1.432766e+18]
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
         expected = DatetimeIndex(['2015-06-19 05:33:20',
                                   '2015-05-27 22:33:20',
                                   'NaT',
                                   'NaT'])
         arr = [1.434692e+18, 1.432766e+18, 'foo', 'NaT']
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
     def test_unit_mixed(self):
@@ -441,17 +422,17 @@ def test_unit_mixed(self):
         tm.assert_index_equal(result, expected)
 
         with pytest.raises(ValueError):
-            pd.to_datetime(arr, errors='raise')
+            pd.to_datetime(arr, unit='ns', errors='raise')
 
         expected = DatetimeIndex(['NaT',
                                   'NaT',
                                   '2013-01-01'])
         arr = [1.434692e+18, 1.432766e+18, pd.Timestamp('20130101')]
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
         with pytest.raises(ValueError):
-            pd.to_datetime(arr, errors='raise')
+            pd.to_datetime(arr, unit='ns', errors='raise')
 
     def test_dataframe(self):
 
@@ -676,36 +657,6 @@ def test_to_datetime_with_apply(self):
             lambda x: pd.to_datetime(x, format='%b %y', errors='coerce'))
         assert_series_equal(result, expected)
 
-    def test_to_datetime_types(self):
-
-        # empty string
-        result = to_datetime('')
-        assert result is NaT
-
-        result = to_datetime(['', ''])
-        assert isnull(result).all()
-
-        # ints
-        result = Timestamp(0)
-        expected = to_datetime(0)
-        assert result == expected
-
-        # GH 3888 (strings)
-        expected = to_datetime(['2012'])[0]
-        result = to_datetime('2012')
-        assert result == expected
-
-        # array = ['2012','20120101','20120101 12:01:01']
-        array = ['20120101', '20120101 12:01:01']
-        expected = list(to_datetime(array))
-        result = lmap(Timestamp, array)
-        tm.assert_almost_equal(result, expected)
-
-        # currently fails ###
-        # result = Timestamp('2012')
-        # expected = to_datetime('2012')
-        # assert result == expected
-
     def test_to_datetime_unprocessable_input(self):
         # GH 4928
         tm.assert_numpy_array_equal(
@@ -1517,6 +1468,16 @@ def julian_dates():
     return pd.date_range('2014-1-1', periods=10).to_julian_date().values
 
 
+@pytest.fixture(params=[True, False])
+def bool_values(request):
+    return request.param
+
+
+@pytest.fixture(params=['coerce', 'ignore'])
+def errors_values(request):
+    return request.param
+
+
 class TestOrigin(object):
 
     def test_to_basic(self, julian_dates):
@@ -1604,3 +1565,63 @@ def test_processing_order(self):
         result = pd.to_datetime(300 * 365, unit='D', origin='1870-01-01')
         expected = Timestamp('2169-10-20 00:00:00')
         assert result == expected
+
+    def test_datetime_dataframe_with_unit(self, units):
+        df = DataFrame({'year': [2000, 2001],
+                        'month': [1.5, 1],
+                        'day': [1, 1]})
+        with pytest.raises(ValueError):
+            to_datetime(df, unit=units)
+
+    def test_datetime_bool(self, units, bool_values, errors_values):
+        # GH13176
+        with pytest.raises(TypeError):
+            to_datetime(bool_values, unit=units)
+        with pytest.raises(TypeError):
+            to_datetime(bool_values, unit=units, errors=errors_values)
+        with pytest.raises(TypeError):
+            to_datetime([bool_values, datetime.today()], unit=units)
+        with pytest.raises(TypeError):
+            to_datetime(['20130101', bool_values])
+
+        tm.assert_index_equal(to_datetime([0, bool_values, NaT, 0.0],
+                                          errors="coerce"),
+                              DatetimeIndex([to_datetime(0, unit=units), NaT,
+                                             NaT, to_datetime(0, unit=units)]))
+
+    def test_to_datetime_types(self, units):
+
+        # empty string
+        result = to_datetime('')
+        assert result is NaT
+
+        result = to_datetime(['', ''])
+        assert(isnull(result).all())
+
+        # ints
+        result = Timestamp(0)
+        expected = to_datetime(0, unit=units)
+        assert result == expected
+
+        # GH 3888 (strings)
+        expected = to_datetime(['2012'])[0]
+        result = to_datetime('2012')
+        assert result == expected
+
+        # array = ['2012','20120101','20120101 12:01:01']
+        array = ['20120101', '20120101 12:01:01']
+        expected = list(to_datetime(array))
+        result = lmap(Timestamp, array)
+        tm.assert_almost_equal(result, expected)
+
+        # currently fails ###
+        # result = Timestamp('2012')
+        # expected = to_datetime('2012')
+        # assert result == expected
+
+    @pytest.mark.parametrize('arg', [int(1), float(1), range(5),
+                                     np.array(range(5), 'd')])
+    def test_to_datetime_numerical_input(self, arg):
+        # GH15836
+        with pytest.raises(ValueError):
+            pd.to_datetime(arg)

pandas/tests/test_resample.py

@@ -1694,7 +1694,7 @@ def test_nanosecond_resample_error(self):
         start = 1443707890427
         exp_start = 1443707890400
         indx = pd.date_range(
-            start=pd.to_datetime(start),
+            start=pd.to_datetime(start, unit='ns'),
             periods=10,
             freq='100n'
         )
@@ -1703,7 +1703,7 @@ def test_nanosecond_resample_error(self):
         result = r.agg('mean')
 
         exp_indx = pd.date_range(
-            start=pd.to_datetime(exp_start),
+            start=pd.to_datetime(exp_start, unit='ns'),
             periods=10,
             freq='100n'
         )