Skip to content

Commit 95e1b84

Browse files
wesmwesm
committed
ENH: encode datetime objects as nanosecond timestamps using numpy machinery, close #1304 
1 parent 9f71e0f commit 95e1b84

File tree

10 files changed

+64
-49
lines changed

10 files changed

+64
-49
lines changed

‎pandas/src/datetime.pxd

+2-2
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ cdef extern from "numpy/npy_common.h":
5757

5858
ctypedef unsigned char npy_bool
5959

60-
cdef extern from "np_datetime.h":
60+
cdef extern from "datetime/np_datetime.h":
6161

6262
ctypedef enum PANDAS_DATETIMEUNIT:
6363
PANDAS_FR_Y
@@ -95,7 +95,7 @@ cdef extern from "np_datetime.h":
9595
int is_leapyear(int64_t year)
9696
PANDAS_DATETIMEUNIT get_datetime64_unit(object o)
9797

98-
cdef extern from "np_datetime_strings.h":
98+
cdef extern from "datetime/np_datetime_strings.h":
9999

100100
int parse_iso_8601_datetime(char *str, int len, PANDAS_DATETIMEUNIT unit,
101101
NPY_CASTING casting, pandas_datetimestruct *out,

‎pandas/src/np_datetime.c renamed to ‎pandas/src/datetime/np_datetime.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
/* #define localtime _localtime64 */
1616
/* #define time _time64 */
1717

18-
#include <numpy/ndarrayobject.h>
18+
#include <numpy/arrayobject.h>
1919
#include <numpy/arrayscalars.h>
2020
#include "np_datetime.h"
2121

‎pandas/src/np_datetime.h renamed to ‎pandas/src/datetime/np_datetime.h

File renamed without changes.

‎pandas/src/np_datetime_strings.c renamed to ‎pandas/src/datetime/np_datetime_strings.c

File renamed without changes.

‎pandas/src/np_datetime_strings.h renamed to ‎pandas/src/datetime/np_datetime_strings.h

File renamed without changes.

‎pandas/src/numpy_helper.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#include "Python.h"
2-
#include "numpy/ndarrayobject.h"
2+
#include "numpy/arrayobject.h"
33
#include "numpy/arrayscalars.h"
44

55
#ifndef PANDAS_INLINE

‎pandas/src/stdint.h

+5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
#ifndef _PANDAS_STDINT_H_
2+
#define _PANDAS_STDINT_H_
3+
14
#if defined(_MSC_VER)
25
#include "ms_stdint.h"
36
#else
47
#include <stdint.h>
58
#endif
9+
10+
#endif

‎pandas/src/ujson/python/objToJSON.c

+18-20
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
#include <py_defines.h>
2-
31
#define PY_ARRAY_UNIQUE_SYMBOL UJSON_NUMPY
42

3+
#include <Python.h>
54
#include <numpy/arrayobject.h>
5+
#include <np_datetime.h>
66
#include <numpy/halffloat.h>
77
#include <stdio.h>
88
#include <datetime.h>
@@ -190,33 +190,31 @@ static void *NpyDateTimeToINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
190190

191191
static void *PyDateTimeToINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
192192
{
193+
pandas_datetimestruct dts;
193194
PyObject *obj = (PyObject *) _obj;
194-
int y, m, d, h, mn, s, days;
195-
196-
y = PyDateTime_GET_YEAR(obj);
197-
m = PyDateTime_GET_MONTH(obj);
198-
d = PyDateTime_GET_DAY(obj);
199-
h = PyDateTime_DATE_GET_HOUR(obj);
200-
mn = PyDateTime_DATE_GET_MINUTE(obj);
201-
s = PyDateTime_DATE_GET_SECOND(obj);
202195

203-
days = PyInt_AS_LONG(PyObject_CallMethod(PyDate_FromDate(y, m, 1), "toordinal", NULL)) - EPOCH_ORD + d - 1;
204-
*( (JSINT64 *) outValue) = (((JSINT64) ((days * 24 + h) * 60 + mn)) * 60 + s);
196+
dts.year = PyDateTime_GET_YEAR(obj);
197+
dts.month = PyDateTime_GET_MONTH(obj);
198+
dts.day = PyDateTime_GET_DAY(obj);
199+
dts.hour = PyDateTime_DATE_GET_HOUR(obj);
200+
dts.min = PyDateTime_DATE_GET_MINUTE(obj);
201+
dts.sec = PyDateTime_DATE_GET_SECOND(obj);
202+
dts.us = PyDateTime_DATE_GET_MICROSECOND(obj);
203+
dts.ps = dts.as = 0;
204+
*((JSINT64*)outValue) = (JSINT64) pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts);
205205
return NULL;
206206
}
207207

208208
static void *PyDateToINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
209209
{
210+
pandas_datetimestruct dts;
210211
PyObject *obj = (PyObject *) _obj;
211-
int y, m, d, days;
212-
213-
y = PyDateTime_GET_YEAR(obj);
214-
m = PyDateTime_GET_MONTH(obj);
215-
d = PyDateTime_GET_DAY(obj);
216-
217-
days = PyInt_AS_LONG(PyObject_CallMethod(PyDate_FromDate(y, m, 1), "toordinal", NULL)) - EPOCH_ORD + d - 1;
218-
*( (JSINT64 *) outValue) = ((JSINT64) days * 86400);
219212

213+
dts.year = PyDateTime_GET_YEAR(obj);
214+
dts.month = PyDateTime_GET_MONTH(obj);
215+
dts.day = PyDateTime_GET_DAY(obj);
216+
dts.hour = dts.min = dts.sec = dts.ps = dts.as = 0;
217+
*((JSINT64*)outValue) = (JSINT64) pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts);
220218
return NULL;
221219
}
222220

‎pandas/tests/test_ujson.py

+29-20
Original file line numberDiff line numberDiff line change
@@ -239,26 +239,35 @@ def test_encodeFalseConversion(self):
239239
self.assertEquals(input, ujson.decode(output))
240240
pass
241241

242-
def test_encodeDatetimeConversion(self):
243-
ts = time.time()
244-
input = datetime.datetime.fromtimestamp(ts)
245-
output = ujson.encode(input)
246-
expected = calendar.timegm(input.utctimetuple())
247-
self.assertEquals(int(expected), json.loads(output))
248-
self.assertEquals(int(expected), ujson.decode(output))
249-
pass
250-
251-
def test_encodeDateConversion(self):
252-
ts = time.time()
253-
input = datetime.date.fromtimestamp(ts)
254-
255-
output = ujson.encode(input)
256-
tup = ( input.year, input.month, input.day, 0, 0, 0 )
257-
258-
expected = calendar.timegm(tup)
259-
self.assertEquals(int(expected), json.loads(output))
260-
self.assertEquals(int(expected), ujson.decode(output))
261-
pass
242+
# def test_encodeDatetimeConversion(self):
243+
# ts = time.time()
244+
# input = datetime.datetime.fromtimestamp(ts)
245+
# output = ujson.encode(input)
246+
# expected = calendar.timegm(input.utctimetuple())
247+
# self.assertEquals(int(expected), json.loads(output))
248+
# self.assertEquals(int(expected), ujson.decode(output))
249+
# pass
250+
251+
# def test_encodeDateConversion(self):
252+
# ts = time.time()
253+
# input = datetime.date.fromtimestamp(ts)
254+
255+
# output = ujson.encode(input)
256+
# tup = ( input.year, input.month, input.day, 0, 0, 0 )
257+
258+
# expected = calendar.timegm(tup)
259+
# self.assertEquals(int(expected), json.loads(output))
260+
# self.assertEquals(int(expected), ujson.decode(output))
261+
262+
def test_datetime_nanosecond_unit(self):
263+
from datetime import datetime
264+
from pandas._tseries import Timestamp
265+
266+
val = datetime.now()
267+
stamp = Timestamp(val)
268+
269+
roundtrip = ujson.decode(ujson.encode(val))
270+
self.assert_(roundtrip == stamp.value)
262271

263272
def test_encodeToUTF8(self):
264273
input = "\xe6\x97\xa5\xd1\x88"

‎setup.py

+8-5
Original file line numberDiff line numberDiff line change
@@ -366,8 +366,8 @@ def srcpath(name=None, suffix='.pyx', subdir='src'):
366366
depends=tseries_depends + ['pandas/src/numpy_helper.h'],
367367
sources=[srcpath('tseries', suffix=suffix),
368368
'pandas/src/period.c',
369-
'pandas/src/np_datetime.c',
370-
'pandas/src/np_datetime_strings.c'],
369+
'pandas/src/datetime/np_datetime.c',
370+
'pandas/src/datetime/np_datetime_strings.c'],
371371
include_dirs=[np.get_include()],
372372
# pyrex_gdb=True,
373373
# extra_compile_args=['-Wconversion']
@@ -387,18 +387,21 @@ def srcpath(name=None, suffix='.pyx', subdir='src'):
387387
include_dirs=[np.get_include()])
388388

389389
npymath_info = get_info('npymath')
390-
390+
391391
npymath_libdir = npymath_info['library_dirs'][0]
392392
npymath_libdir = npymath_libdir.replace('\\\\', '\\')
393-
393+
394394
ujson_ext = Extension('pandas._ujson',
395395
sources=['pandas/src/ujson/python/ujson.c',
396396
'pandas/src/ujson/python/objToJSON.c',
397397
'pandas/src/ujson/python/JSONtoObj.c',
398398
'pandas/src/ujson/lib/ultrajsonenc.c',
399-
'pandas/src/ujson/lib/ultrajsondec.c'],
399+
'pandas/src/ujson/lib/ultrajsondec.c',
400+
'pandas/src/datetime/np_datetime.c'
401+
],
400402
include_dirs=['pandas/src/ujson/python',
401403
'pandas/src/ujson/lib',
404+
'pandas/src/datetime',
402405
np.get_include()],
403406
libraries=['npymath'],
404407
library_dirs=[npymath_libdir],

0 commit comments

Comments
 (0)