ENH: more frequency inference, unique_deltas function

Author: wesm

pandas/src/datetime.pyx

@@ -41,6 +41,8 @@ ctypedef enum time_res:
 # This serves as the box for datetime64
 class Timestamp(_Timestamp):
 
+    __slots__ = ['value', 'offset']
+
     def __new__(cls, object ts_input, object offset=None, tz=None):
         if isinstance(ts_input, float):
             # to do, do we want to support this, ie with fractional seconds?
@@ -103,6 +105,37 @@ class Timestamp(_Timestamp):
 
         return Period(self, freq=freq)
 
+#----------------------------------------------------------------------
+# Frequency inference
+
+def unique_deltas(ndarray[int64_t] arr):
+    cdef:
+        Py_ssize_t i, n = len(arr)
+        int64_t val
+        khiter_t k
+        kh_int64_t *table
+        int ret = 0
+        list uniques = []
+
+    table = kh_init_int64()
+    kh_resize_int64(table, 10)
+    for i in range(n - 1):
+        val = arr[i + 1] - arr[i]
+        k = kh_get_int64(table, val)
+        if k == table.n_buckets:
+            kh_put_int64(table, val, &ret)
+            uniques.append(val)
+    kh_destroy_int64(table)
+
+    result = np.array(uniques, dtype=np.int64)
+    result.sort()
+    return result
+
+
+cdef inline bint _is_multiple(int64_t us, int64_t mult):
+    return us % mult == 0
+
+
 def apply_offset(ndarray[object] values, object offset):
     cdef:
         Py_ssize_t i, n = len(values)
@@ -843,7 +876,7 @@ def tz_localize_array(ndarray[int64_t] vals, object tz):
     return vals
 
 # Accessors
-# ------------------------------------------------------------------------------
+#----------------------------------------------------------------------
 
 def build_field_sarray(ndarray[int64_t] dtindex):
     '''
@@ -966,8 +999,14 @@ def fast_field_accessor(ndarray[int64_t] dtindex, object field):
 
     raise ValueError("Field %s not supported" % field)
 
+
+cdef inline int m8_weekday(int64_t val):
+    ts = convert_to_tsobject(val)
+    return ts_dayofweek(ts)
+
+
 # Some general helper functions
-# ------------------------------------------------------------------------------
+#----------------------------------------------------------------------
 
 def isleapyear(int64_t year):
     return is_leapyear(year)
@@ -988,7 +1027,7 @@ cdef inline int64_t ts_dayofweek(_TSObject ts):
     return dayofweek(ts.dtval.year, ts.dtval.month, ts.dtval.day)
 
 # Period logic
-# ------------------------------------------------------------------------------
+#----------------------------------------------------------------------
 
 cdef int64_t apply_mult(int64_t period_ord, int64_t mult):
     """

pandas/src/khash.h

@@ -114,7 +114,6 @@ int main() {
 #include <limits.h>
 #include <Python.h>
 
-/* compipler specific configuration */
 
 #if UINT_MAX == 0xffffffffu
 typedef unsigned int khint32_t;

pandas/src/ktypes.h

@@ -0,0 +1,6 @@
+#ifndef __KTYPES_H
+#define __KTYPES_H
+
+/* compipler specific configuration */
+
+#endif /* __KTYPES_H */

pandas/src/kvec.h

@@ -0,0 +1,98 @@
+/* The MIT License
+
+   Copyright (c) 2008, by Attractive Chaos <[email protected]>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "kvec.h"
+int main() {
+	kvec_t(int) array;
+	kv_init(array);
+	kv_push(int, array, 10); // append
+	kv_a(int, array, 20) = 5; // dynamic
+	kv_A(array, 20) = 4; // static
+	kv_destroy(array);
+	return 0;
+}
+*/
+
+/*
+  2008-09-22 (0.1.0):
+
+	* The initial version.
+
+*/
+
+#ifndef AC_KVEC_H
+#define AC_KVEC_H
+
+#include <stdlib.h>
+#include <ktypes.h>
+
+#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+#define kv_init(v) ((v).n = (v).m = 0, (v).a = 0)
+#define kv_destroy(v) free((v).a)
+#define kv_A(v, i) ((v).a[(i)])
+#define kv_pop(v) ((v).a[--(v).n])
+#define kv_size(v) ((v).n)
+#define kv_max(v) ((v).m)
+
+#define kv_resize(type, v, s)  ((v).m = (s), (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_copy(type, v1, v0) do {							\
+		if ((v1).m < (v0).n) kv_resize(type, v1, (v0).n);	\
+		(v1).n = (v0).n;									\
+		memcpy((v1).a, (v0).a, sizeof(type) * (v0).n);		\
+	} while (0)												\
+
+#define kv_push(type, v, x) do {									\
+		if ((v).n == (v).m) {										\
+			(v).m = (v).m? (v).m<<1 : 2;							\
+			(v).a = (type*)realloc((v).a, sizeof(type) * (v).m);	\
+		}															\
+		(v).a[(v).n++] = (x);										\
+	} while (0)
+
+#define kv_pushp(type, v) (((v).n == (v).m)?							\
+						   ((v).m = ((v).m? (v).m<<1 : 2),				\
+							(v).a = (type*)realloc((v).a, sizeof(type) * (v).m), 0)	\
+						   : 0), ((v).a + ((v).n++))
+
+#define kv_a(type, v, i) ((v).m <= (size_t)(i)?						\
+						  ((v).m = (v).n = (i) + 1, kv_roundup32((v).m), \
+						   (v).a = (type*)realloc((v).a, sizeof(type) * (v).m), 0) \
+						  : (v).n <= (size_t)(i)? (v).n = (i)			\
+						  : 0), (v).a[(i)]
+
+#define kv_int64_push(v, x) (kv_push(int64_t, (v), (x)))
+
+typedef struct {
+  size_t n, m;
+  int64_t *a;
+} kv_int64_t;
+
+#endif

pandas/src/sandbox.pyx

@@ -533,6 +533,13 @@ def group_add(ndarray[float64_t, ndim=2] out,
 from datetime cimport getAbsTime
 
 
+# cdef extern from "kvec.h":
+
+#     ctypedef struct kv_int64_t:
+#         size_t n, m
+#         int64_t *a
+
+
+
 def get_abs_time(freq, dailyDate, originalDate):
     return getAbsTime(freq, dailyDate, originalDate)
-

pandas/tseries/frequencies.py

@@ -1,6 +1,7 @@
 import re
 
 from pandas.tseries.offsets import DateOffset
+from pandas._tseries import Timestamp
 import pandas.tseries.offsets as offsets
 
 
@@ -52,7 +53,7 @@ def _get_freq_str(base, mult):
 _unknown_freq = 'Unknown'
 
 
-#-------------------------------------------------------------------------------
+#----------------------------------------------------------------------
 # Offset names ("time rules") and related functions
 
 
@@ -615,3 +616,142 @@ def _period_str_to_code(freqstr):
             return _period_code_map[alias]
         except:
             raise "Could not interpret frequency %s" % freqstr
+
+
+def infer_freq(index):
+    """
+    Not sure if I can avoid the state machine here
+    """
+    from pandas._sandbox import unique_deltas
+
+    if len(index) < 3:
+        raise ValueError('Need at least 3 dates to infer frequency')
+
+    deltas = unique_deltas(index)
+
+    is_unique = len(deltas) == 1
+
+    if _is_multiple(deltas[0], _day_us):
+        if is_unique:
+            days = deltas[0] / _day_us
+            if days % 7 == 0:
+                # Weekly
+                alias = _weekday_rule_aliases[days]
+                return _maybe_add_count('W-%s' % alias, days / 7)
+            else:
+                return _maybe_add_count('D', days)
+
+        fields = lib.build_field_sarray(index)
+
+        day_list = [x / _day_us for x in deltas]
+        rstamp = Timestamp(index[0])
+
+        annual_rule = _get_annual_rule(fields)
+        if annual_rule:
+            nyears = day_list[0] // 365
+            month = _month_aliases[rstamp.month]
+            return _maybe_add_count('%s-%s' % (annual_rule, month), nyears)
+
+        quarterly_rule = _get_quarterly_rule(fields)
+        if quarterly_rule:
+            month = _month_aliases[rstamp.month]
+            return '%s-%s' % (quarterly_rule, month)
+
+        elif _is_quarterly_deltas(day_list):
+            pass
+        elif _is_monthly_deltas(day_list):
+            pass
+        else:
+            # Business daily. Maybe
+            pass
+
+    elif _is_multiple(deltas[0], 60 * 60 * 1000000):
+        if not is_unique:
+            return None
+        # Hours
+        return '%dH' % (deltas[0] / (60 * 60 * 1000000))
+    elif _is_multiple(deltas[0], 60 * 1000000):
+        if not is_unique:
+            return None
+        # Minutes
+        return '%dT' % (deltas[0] / (60 * 1000000))
+    elif _is_multiple(deltas[0], 1000000):
+        if not is_unique:
+            return None
+        # Seconds
+        return '%dS' % (deltas[0] / 1000000)
+    elif _is_multiple(deltas[0], 1000):
+        if not is_unique:
+            return None
+        # Milliseconds
+        return '%dL' % (deltas[0] / 1000)
+    else:
+        if not is_unique:
+            return None
+        # Microseconds
+        return '%dU' % deltas[0]
+
+
+import pandas.core.algorithms as algos
+
+
+def _get_annual_rule(fields):
+    years = fields['Y']
+    months = fields['M']
+    days = fields['D']
+
+    ydiffs = unique_deltas(years.astype('i8'))
+    if len(ydiffs) > 1:
+        return False
+
+    if len(algos.unique(months)) == 1:
+        if _all_last_weekday(years, months, days):
+            return
+
+
+def _is_quarterly_deltas(day_list):
+    pass
+
+def _is_monthly_deltas(day_list):
+    pass
+
+def _is_business_years(index):
+    pass
+
+def _maybe_add_count(base, count):
+    if count > 1:
+        return '%d%s' % (count, base)
+    else:
+        return base
+
+
+
+_weekday_rule_aliases = {
+    0: 'MON',
+    1: 'TUE',
+    2: 'WED',
+    3: 'THU',
+    4: 'FRI',
+    5: 'SAT',
+    6: 'SUN'
+}
+
+_month_aliases = {
+    1: 'JAN',
+    2: 'FEB',
+    3: 'MAR',
+    4: 'APR',
+    5: 'MAY',
+    6: 'JUN',
+    7: 'JUL',
+    8: 'AUG',
+    9: 'SEP',
+    10: 'OCT',
+    11: 'NOV',
+    12: 'DEC'
+}
+
+def _is_multiple(us, mult):
+    return us % mult == 0
+
+_day_us = 24 * 60 * 60 * 1000000

pandas/tseries/index.py

@@ -4,6 +4,7 @@
 import numpy as np
 
 from pandas.core.index import Index, Int64Index
+from pandas.tseries.frequencies import infer_freq
 from pandas.tseries.tools import parse_time_string
 from pandas.util.decorators import cache_readonly
 import pandas.core.common as com
@@ -841,8 +842,7 @@ def freq(self):
 
     @cache_readonly
     def inferred_freq(self):
-        import pandas._sandbox as sbx
-        return sbx.infer_freq(self.asi8)
+        return infer_freq(self.asi8)
 
     @property
     def freqstr(self):