REF: more nanosecond support fixes, test suite passes #1238

Author: wesm

pandas/core/algorithms.py

@@ -108,6 +108,8 @@ def factorize(values, sort=False, order=None, na_sentinel=-1):
     Returns
     -------
     """
+    values = np.asarray(values)
+    is_datetime = com.is_datetime64_dtype(values)
     hash_klass, values = _get_data_algo(values, _hashtables)
 
     uniques = []
@@ -129,6 +131,9 @@ def factorize(values, sort=False, order=None, na_sentinel=-1):
         uniques = uniques.take(sorter)
         counts = counts.take(sorter)
 
+    if is_datetime:
+        uniques = np.array(uniques, dtype='M8[ns]')
+
     return labels, uniques, counts
 
 def value_counts(values, sort=True, ascending=False):
@@ -179,6 +184,9 @@ def _get_data_algo(values, func_map):
     if com.is_float_dtype(values):
         f = func_map['float64']
         values = com._ensure_float64(values)
+    elif com.is_datetime64_dtype(values):
+        f = func_map['int64']
+        values = values.view('i8')
     elif com.is_integer_dtype(values):
         f = func_map['int64']
         values = com._ensure_int64(values)

pandas/core/common.py

@@ -171,7 +171,7 @@ def wrapper(arr, indexer, out, fill_value=np.nan):
     'int64' : _algos.take_1d_int64,
     'object' : _algos.take_1d_object,
     'bool' : _view_wrapper(_algos.take_1d_bool, np.uint8),
-    'datetime64[us]' : _view_wrapper(_algos.take_1d_int64, np.int64,
+    'datetime64[ns]' : _view_wrapper(_algos.take_1d_int64, np.int64,
                                      na_override=lib.NaT),
 }
 
@@ -181,7 +181,7 @@ def wrapper(arr, indexer, out, fill_value=np.nan):
     'int64' : _algos.take_2d_axis0_int64,
     'object' : _algos.take_2d_axis0_object,
     'bool' : _view_wrapper(_algos.take_2d_axis0_bool, np.uint8),
-    'datetime64[us]' : _view_wrapper(_algos.take_2d_axis0_int64, np.int64,
+    'datetime64[ns]' : _view_wrapper(_algos.take_2d_axis0_int64, np.int64,
                                      na_override=lib.NaT),
 }
 
@@ -191,7 +191,7 @@ def wrapper(arr, indexer, out, fill_value=np.nan):
     'int64' : _algos.take_2d_axis1_int64,
     'object' : _algos.take_2d_axis1_object,
     'bool' : _view_wrapper(_algos.take_2d_axis1_bool, np.uint8),
-    'datetime64[us]' : _view_wrapper(_algos.take_2d_axis1_int64, np.int64,
+    'datetime64[ns]' : _view_wrapper(_algos.take_2d_axis1_int64, np.int64,
                                      na_override=lib.NaT),
 }
 
@@ -201,7 +201,7 @@ def wrapper(arr, indexer, out, fill_value=np.nan):
     'int64' : _algos.take_2d_multi_int64,
     'object' : _algos.take_2d_multi_object,
     'bool' : _view_wrapper(_algos.take_2d_multi_bool, np.uint8),
-    'datetime64[us]' : _view_wrapper(_algos.take_2d_multi_int64, np.int64,
+    'datetime64[ns]' : _view_wrapper(_algos.take_2d_multi_int64, np.int64,
                                      na_override=lib.NaT),
 }
 
@@ -246,7 +246,7 @@ def take_1d(arr, indexer, out=None, fill_value=np.nan):
                                     out.dtype)
                 out = _maybe_upcast(out)
                 np.putmask(out, mask, fill_value)
-    elif dtype_str in ('float64', 'object', 'datetime64[us]'):
+    elif dtype_str in ('float64', 'object', 'datetime64[ns]'):
         if out is None:
             out = np.empty(n, dtype=arr.dtype)
         take_f(arr, _ensure_int64(indexer), out=out, fill_value=fill_value)
@@ -284,7 +284,7 @@ def take_2d_multi(arr, row_idx, col_idx, fill_value=np.nan):
                    _ensure_int64(col_idx), out=out,
                    fill_value=fill_value)
             return out
-    elif dtype_str in ('float64', 'object', 'datetime64[us]'):
+    elif dtype_str in ('float64', 'object', 'datetime64[ns]'):
         out = np.empty(out_shape, dtype=arr.dtype)
         take_f(arr, _ensure_int64(row_idx), _ensure_int64(col_idx), out=out,
                fill_value=fill_value)
@@ -326,7 +326,7 @@ def take_2d(arr, indexer, out=None, mask=None, needs_masking=None, axis=0,
             take_f = _get_take2d_function(dtype_str, axis=axis)
             take_f(arr, _ensure_int64(indexer), out=out, fill_value=fill_value)
             return out
-    elif dtype_str in ('float64', 'object', 'datetime64[us]'):
+    elif dtype_str in ('float64', 'object', 'datetime64[ns]'):
         if out is None:
             out = np.empty(out_shape, dtype=arr.dtype)
         take_f = _get_take2d_function(dtype_str, axis=axis)

pandas/core/factor.py

@@ -18,11 +18,17 @@ class Factor(np.ndarray):
       * levels : ndarray
     """
     def __new__(cls, data):
-        data = np.asarray(data, dtype=object)
-        levels, factor = unique_with_labels(data)
-        factor = factor.view(Factor)
-        factor.levels = levels
-        return factor
+        from pandas.core.index import _ensure_index
+        from pandas.core.algorithms import factorize
+
+        try:
+            labels, levels, _ = factorize(data, sort=True)
+        except TypeError:
+            labels, levels, _ = factorize(data, sort=False)
+
+        labels = labels.view(Factor)
+        labels.levels = _ensure_index(levels)
+        return labels
 
     levels = None
 
@@ -51,21 +57,3 @@ def __getitem__(self, key):
         else:
             return np.ndarray.__getitem__(self, key)
 
-
-def unique_with_labels(values):
-    from pandas.core.index import Index
-    rizer = lib.Factorizer(len(values))
-    labels, _ = rizer.factorize(values, sort=False)
-    uniques = Index(rizer.uniques)
-    labels = com._ensure_platform_int(labels)
-    try:
-        sorter = uniques.argsort()
-        reverse_indexer = np.empty(len(sorter), dtype=np.int_)
-        reverse_indexer.put(sorter, np.arange(len(sorter)))
-        labels = reverse_indexer.take(labels)
-        uniques = uniques.take(sorter)
-    except TypeError:
-        pass
-
-    return uniques, labels
-

pandas/core/format.py

@@ -571,16 +571,30 @@ def get_result(self):
         if self.formatter:
             formatter = self.formatter
         else:
-            def formatter(x):
-                if isnull(x):
-                    return 'NaT'
-                else:
-                    return str(x)
+            formatter = _format_datetime64
 
         fmt_values = [formatter(x) for x in self.values]
-
         return _make_fixed_width(fmt_values, self.justify)
 
+def _format_datetime64(x):
+    if isnull(x):
+        return 'NaT'
+
+    stamp = lib.Timestamp(x)
+    base = stamp.strftime('%Y-%m-%d %H:%M:%S')
+
+    fraction = stamp.microsecond * 1000 + stamp.nanosecond
+    digits = 9
+
+    if fraction == 0:
+        return base
+
+    while (fraction % 10) == 0:
+        fraction /= 10
+        digits -= 1
+
+    return base + ('.%%.%id' % digits) % fraction
+
 
 def _make_fixed_width(strings, justify='right'):
     if len(strings) == 0:

pandas/core/index.py

@@ -691,8 +691,8 @@ def get_indexer(self, target, method=None, limit=None):
             return pself.get_indexer(ptarget, method=method, limit=limit)
 
         if self.dtype != target.dtype:
-            this = Index(self, dtype=object)
-            target = Index(target, dtype=object)
+            this = self.astype(object)
+            target = target.astype(object)
             return this.get_indexer(target, method=method, limit=limit)
 
         if not self.is_unique:
@@ -1172,8 +1172,12 @@ def __new__(cls, levels=None, labels=None, sortorder=None, names=None):
         levels = [_ensure_index(lev) for lev in levels]
         labels = [np.asarray(labs, dtype=np.int_) for labs in labels]
 
-        values = [ndtake(np.asarray(lev), lab)
+        values = [ndtake(lev.values, lab)
                   for lev, lab in zip(levels, labels)]
+
+        # Need to box timestamps, etc.
+        values = _clean_arrays(values)
+
         subarr = lib.fast_zip(values).view(cls)
 
         subarr.levels = levels
@@ -2372,3 +2376,13 @@ def _maybe_box_dtindex(idx):
         return Index(_dt_box_array(idx.asi8), dtype='object')
     return idx
 
+def _clean_arrays(values):
+    result = []
+    for arr in values:
+        if np.issubdtype(arr.dtype, np.datetime_):
+            result.append(lib.map_infer(arr, lib.Timestamp))
+        else:
+            result.append(arr)
+    return result
+
+

pandas/core/nanops.py

@@ -405,7 +405,7 @@ def unique1d(values):
         uniques = np.array(table.unique(com._ensure_int64(values)),
                            dtype=np.int64)
 
-        if values.dtype == np.datetime64:
+        if issubclass(values.dtype.type, np.datetime_):
             uniques = uniques.view('M8[ns]')
     else:
         table = lib.PyObjectHashTable(len(values))

pandas/io/pytables.py

@@ -839,8 +839,7 @@ def _read_panel_table(self, group, where=None):
 
         columns = _maybe_convert(sel.values['column'],
                                  table._v_attrs.columns_kind)
-        index = _maybe_convert(sel.values['index'],
-                               table._v_attrs.index_kind)
+        index = _maybe_convert(sel.values['index'], table._v_attrs.index_kind)
         values = sel.values['values']
 
         major = Factor(index)
@@ -995,7 +994,7 @@ def _maybe_convert(values, val_kind):
 
 def _get_converter(kind):
     if kind == 'datetime64':
-        return lambda x: np.datetime64(x)
+        return lambda x: np.array(x, dtype='M8[ns]')
     if kind == 'datetime':
         return lib.convert_timestamps
     else: # pragma: no cover
@@ -1069,7 +1068,7 @@ def generate(self, where):
             field = c['field']
 
             if field == 'index' and self.index_kind == 'datetime64':
-                val = np.datetime64(value).view('i8')
+                val = lib.Timestamp(value).value
                 self.conditions.append('(%s %s %s)' % (field,op,val))
             elif field == 'index' and isinstance(value, datetime):
                 value = time.mktime(value.timetuple())

pandas/io/tests/test_parsers.py

@@ -376,7 +376,8 @@ def test_parse_dates_column_list(self):
         lev = expected.index.levels[0]
         expected.index.levels[0] = lev.to_datetime(dayfirst=True)
         expected['aux_date'] = to_datetime(expected['aux_date'],
-                                           dayfirst=True).astype('O')
+                                           dayfirst=True)
+        expected['aux_date'] = map(Timestamp, expected['aux_date'])
         self.assert_(isinstance(expected['aux_date'][0], datetime))
 
         df = read_csv(StringIO(data), sep=";", index_col = range(4),

pandas/sparse/frame.py

@@ -741,6 +741,23 @@ def apply(self, func, axis=0, broadcast=False):
             else:
                 return self._apply_broadcast(func, axis)
 
+    def applymap(self, func):
+        """
+        Apply a function to a DataFrame that is intended to operate
+        elementwise, i.e. like doing map(func, series) for each series in the
+        DataFrame
+
+        Parameters
+        ----------
+        func : function
+            Python function, returns a single value from a single value
+
+        Returns
+        -------
+        applied : DataFrame
+        """
+        return self.apply(lambda x: map(func, x))
+
     @Appender(DataFrame.fillna.__doc__)
     def fillna(self, value=None, method='pad', inplace=False, limit=None):
         new_series = {}

pandas/src/datetime.pyx

@@ -136,6 +136,11 @@ class Timestamp(_Timestamp):
         conv = tz.normalize(self)
         return Timestamp(conv)
 
+    def replace(self, **kwds):
+        return Timestamp(datetime.replace(self, **kwds),
+                         offset=self.offset)
+
+
 cdef inline bint is_timestamp(object o):
     return isinstance(o, Timestamp)
 
@@ -194,10 +199,38 @@ def apply_offset(ndarray[object] values, object offset):
 # (see Timestamp class above). This will serve as a C extension type that
 # shadows the python class, where we do any heavy lifting.
 cdef class _Timestamp(datetime):
-    cdef public:
+    cdef readonly:
         int64_t value, nanosecond
         object offset       # frequency reference
 
+    def __richcmp__(_Timestamp self, object other, int op):
+        cdef _Timestamp ots
+
+        if isinstance(other, _Timestamp):
+            ots = other
+        elif isinstance(other, datetime):
+            ots = Timestamp(other)
+        else:
+            if op == 2:
+                return False
+            elif op == 3:
+                return True
+            else:
+                raise TypeError('Cannot compare Timestamp with %s' % str(other))
+
+        if op == 2: # ==
+            return self.value == ots.value
+        elif op == 3: # !=
+            return self.value != ots.value
+        elif op == 0: # <
+            return self.value < ots.value
+        elif op == 1: # <=
+            return self.value <= ots.value
+        elif op == 4: # >
+            return self.value > ots.value
+        elif op == 5: # >=
+            return self.value >= ots.value
+
     def __add__(self, other):
         if is_integer_object(other):
             if self.offset is None:
@@ -313,6 +346,7 @@ cdef inline int64_t _pydatetime_to_dts(object val, pandas_datetimestruct *dts):
     dts.min = PyDateTime_DATE_GET_MINUTE(val)
     dts.sec = PyDateTime_DATE_GET_SECOND(val)
     dts.us = PyDateTime_DATE_GET_MICROSECOND(val)
+    dts.ps = dts.as = 0
     return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts)
 
 cdef inline int64_t _dtlike_to_datetime64(object val,
@@ -324,17 +358,16 @@ cdef inline int64_t _dtlike_to_datetime64(object val,
     dts.min = val.minute
     dts.sec = val.second
     dts.us = val.microsecond
+    dts.ps = dts.as = 0
     return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts)
 
 cdef inline int64_t _date_to_datetime64(object val,
                                         pandas_datetimestruct *dts):
     dts.year = PyDateTime_GET_YEAR(val)
     dts.month = PyDateTime_GET_MONTH(val)
     dts.day = PyDateTime_GET_DAY(val)
-    dts.hour = 0
-    dts.min = 0
-    dts.sec = 0
-    dts.us = 0
+    dts.hour = dts.min = dts.sec = dts.us = 0
+    dts.ps = dts.as = 0
     return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, dts)
 
 
@@ -928,7 +961,7 @@ cpdef ndarray _unbox_utcoffsets(object transinfo):
     arr = np.empty(sz, dtype='i8')
 
     for i in range(sz):
-        arr[i] = int(total_seconds(transinfo[i][0])) * 1000000
+        arr[i] = int(total_seconds(transinfo[i][0])) * 1000000000
 
     return arr
 
@@ -1243,7 +1276,7 @@ def dt64arr_to_periodarr(ndarray[int64_t] dtarr, int freq):
     for i in range(l):
         pandas_datetime_to_datetimestruct(dtarr[i], PANDAS_FR_ns, &dts)
         out[i] = get_period_ordinal(dts.year, dts.month, dts.day,
-                                  dts.hour, dts.min, dts.sec, freq)
+                                    dts.hour, dts.min, dts.sec, freq)
     return out
 
 def periodarr_to_dt64arr(ndarray[int64_t] periodarr, int freq):
@@ -1338,7 +1371,7 @@ cpdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq):
     dts.hour = dinfo.hour
     dts.min = dinfo.minute
     dts.sec = int(dinfo.second)
-    dts.us = 0
+    dts.us = dts.ps = 0
 
     return pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts)